2024-09-13T00:05:13.137 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-13T00:05:13.162 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T00:05:13.260 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901110 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' first_in_suite: false job_id: '7901110' kernel: kdb: 1 sha1: distro ktype: distro last_in_suite: false machine_type: smithi meta: - desc: 'librbd C and C++ api tests ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 workunit: branch: reef sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 owner: scheduled_teuthology@teuthology priority: 830 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 4534 sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 0/1 suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/git.ceph.com_ceph_7a853b1706d076d58e0da7a5d3b9b9eb36276880/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 targets: smithi077.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEKXbcXEr5OD7nu3KGF7XHKtcXROnNKD4O1YgwW+mrVw4xqbhASDV/2PLC5aZczGfQzJBEOxuuxezA3I43HKUfM= smithi088.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLsBIvYhpYjW+S3JNMDzftuI/+kKpGA3tTkceW8O1RVLD93Is9j3+Jd2YSW663Emzv20bn18MfDqjRXO9QPq+/k= tasks: - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-11_01:16:05 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-09-13T00:05:13.260 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_7a853b1706d076d58e0da7a5d3b9b9eb36276880/qa; will attempt to use it 2024-09-13T00:05:13.261 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_7a853b1706d076d58e0da7a5d3b9b9eb36276880/qa/tasks 2024-09-13T00:05:13.261 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-13T00:05:13.262 INFO:teuthology.task.internal:Checking packages... 2024-09-13T00:05:13.286 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash '7a853b1706d076d58e0da7a5d3b9b9eb36276880' 2024-09-13T00:05:13.286 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T00:05:13.286 INFO:teuthology.packaging:ref: None 2024-09-13T00:05:13.286 INFO:teuthology.packaging:tag: None 2024-09-13T00:05:13.286 INFO:teuthology.packaging:branch: reef 2024-09-13T00:05:13.286 INFO:teuthology.packaging:sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-13T00:05:13.287 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=reef 2024-09-13T00:05:13.520 INFO:teuthology.task.internal:Found packages for ceph version 18.2.4-841-g72e3ed98-1jammy 2024-09-13T00:05:13.522 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-13T00:05:13.550 INFO:teuthology.task.internal:no buildpackages task found 2024-09-13T00:05:13.550 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-13T00:05:13.573 INFO:teuthology.task.internal:Saving configuration 2024-09-13T00:05:13.593 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-13T00:05:13.609 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-13T00:05:13.634 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi077.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901110', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-12 23:59:15.990187', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICygadeuPDA9PX8U+P8Px6l6IAGb2DkPPEQZAY8TPF0f'} 2024-09-13T00:05:13.655 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi088.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901110', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-12 23:59:15.992056', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMkbvbk4h+QX4Zq4Fy30gZaekLNgbhdxR5VA1UdFMjoH'} 2024-09-13T00:05:13.655 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-13T00:05:13.674 INFO:teuthology.task.internal:roles: ubuntu@smithi077.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-13T00:05:13.674 INFO:teuthology.task.internal:roles: ubuntu@smithi088.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-13T00:05:13.674 INFO:teuthology.run_tasks:Running task console_log... 2024-09-13T00:05:13.734 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f35725cb0a0>, signals=[15]) 2024-09-13T00:05:13.734 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-13T00:05:13.753 INFO:teuthology.task.internal:Opening connections... 2024-09-13T00:05:13.753 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi077.front.sepia.ceph.com 2024-09-13T00:05:13.755 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi077.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T00:05:13.833 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi088.front.sepia.ceph.com 2024-09-13T00:05:13.834 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T00:05:13.912 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-13T00:05:13.917 DEBUG:teuthology.orchestra.run.smithi077:> uname -m 2024-09-13T00:05:13.922 INFO:teuthology.orchestra.run.smithi077.stdout:x86_64 2024-09-13T00:05:13.922 DEBUG:teuthology.orchestra.run.smithi077:> cat /etc/os-release 2024-09-13T00:05:13.969 INFO:teuthology.orchestra.run.smithi077.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-09-13T00:05:13.969 INFO:teuthology.orchestra.run.smithi077.stdout:NAME="Ubuntu" 2024-09-13T00:05:13.970 INFO:teuthology.orchestra.run.smithi077.stdout:VERSION_ID="22.04" 2024-09-13T00:05:13.970 INFO:teuthology.orchestra.run.smithi077.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-09-13T00:05:13.970 INFO:teuthology.orchestra.run.smithi077.stdout:VERSION_CODENAME=jammy 2024-09-13T00:05:13.970 INFO:teuthology.orchestra.run.smithi077.stdout:ID=ubuntu 2024-09-13T00:05:13.970 INFO:teuthology.orchestra.run.smithi077.stdout:ID_LIKE=debian 2024-09-13T00:05:13.970 INFO:teuthology.orchestra.run.smithi077.stdout:HOME_URL="https://www.ubuntu.com/" 2024-09-13T00:05:13.970 INFO:teuthology.orchestra.run.smithi077.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-09-13T00:05:13.970 INFO:teuthology.orchestra.run.smithi077.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-09-13T00:05:13.970 INFO:teuthology.orchestra.run.smithi077.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-09-13T00:05:13.970 INFO:teuthology.orchestra.run.smithi077.stdout:UBUNTU_CODENAME=jammy 2024-09-13T00:05:13.971 INFO:teuthology.lock.ops:Updating smithi077.front.sepia.ceph.com on lock server 2024-09-13T00:05:13.990 DEBUG:teuthology.orchestra.run.smithi088:> uname -m 2024-09-13T00:05:13.995 INFO:teuthology.orchestra.run.smithi088.stdout:x86_64 2024-09-13T00:05:13.995 DEBUG:teuthology.orchestra.run.smithi088:> cat /etc/os-release 2024-09-13T00:05:14.041 INFO:teuthology.orchestra.run.smithi088.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-09-13T00:05:14.041 INFO:teuthology.orchestra.run.smithi088.stdout:NAME="Ubuntu" 2024-09-13T00:05:14.041 INFO:teuthology.orchestra.run.smithi088.stdout:VERSION_ID="22.04" 2024-09-13T00:05:14.041 INFO:teuthology.orchestra.run.smithi088.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-09-13T00:05:14.042 INFO:teuthology.orchestra.run.smithi088.stdout:VERSION_CODENAME=jammy 2024-09-13T00:05:14.042 INFO:teuthology.orchestra.run.smithi088.stdout:ID=ubuntu 2024-09-13T00:05:14.042 INFO:teuthology.orchestra.run.smithi088.stdout:ID_LIKE=debian 2024-09-13T00:05:14.042 INFO:teuthology.orchestra.run.smithi088.stdout:HOME_URL="https://www.ubuntu.com/" 2024-09-13T00:05:14.042 INFO:teuthology.orchestra.run.smithi088.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-09-13T00:05:14.043 INFO:teuthology.orchestra.run.smithi088.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-09-13T00:05:14.043 INFO:teuthology.orchestra.run.smithi088.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-09-13T00:05:14.043 INFO:teuthology.orchestra.run.smithi088.stdout:UBUNTU_CODENAME=jammy 2024-09-13T00:05:14.043 INFO:teuthology.lock.ops:Updating smithi088.front.sepia.ceph.com on lock server 2024-09-13T00:05:14.066 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-13T00:05:14.084 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-13T00:05:14.091 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-13T00:05:14.092 DEBUG:teuthology.orchestra.run.smithi077:> test '!' -e /home/ubuntu/cephtest 2024-09-13T00:05:14.095 DEBUG:teuthology.orchestra.run.smithi088:> test '!' -e /home/ubuntu/cephtest 2024-09-13T00:05:14.100 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-13T00:05:14.108 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-13T00:05:14.108 DEBUG:teuthology.orchestra.run.smithi077:> test -z $(ls -A /var/lib/ceph) 2024-09-13T00:05:14.144 DEBUG:teuthology.orchestra.run.smithi088:> test -z $(ls -A /var/lib/ceph) 2024-09-13T00:05:14.153 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-13T00:05:14.217 INFO:teuthology.run_tasks:Running task kernel... 2024-09-13T00:05:14.248 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-13T00:05:14.249 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-13T00:05:14.249 DEBUG:teuthology.orchestra.run.smithi077:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T00:05:14.250 DEBUG:teuthology.orchestra.run.smithi088:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-13T00:05:14.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T00:05:14.254 DEBUG:teuthology.orchestra.run.smithi088:> uname -r 2024-09-13T00:05:14.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T00:05:14.256 DEBUG:teuthology.orchestra.run.smithi077:> uname -r 2024-09-13T00:05:14.301 INFO:teuthology.orchestra.run.smithi088.stdout:5.15.0-56-generic 2024-09-13T00:05:14.301 INFO:teuthology.task.kernel:Running kernel on smithi088: 5.15.0-56-generic 2024-09-13T00:05:14.302 DEBUG:teuthology.orchestra.run.smithi088:> sudo apt-get clean 2024-09-13T00:05:14.303 INFO:teuthology.orchestra.run.smithi077.stdout:5.15.0-56-generic 2024-09-13T00:05:14.303 INFO:teuthology.task.kernel:Running kernel on smithi077: 5.15.0-56-generic 2024-09-13T00:05:14.303 DEBUG:teuthology.orchestra.run.smithi077:> sudo apt-get clean 2024-09-13T00:05:14.433 DEBUG:teuthology.orchestra.run.smithi088:> sudo apt-get update 2024-09-13T00:05:14.446 DEBUG:teuthology.orchestra.run.smithi077:> sudo apt-get update 2024-09-13T00:05:14.655 INFO:teuthology.orchestra.run.smithi088.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-13T00:05:14.656 INFO:teuthology.orchestra.run.smithi077.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-13T00:05:14.663 INFO:teuthology.orchestra.run.smithi077.stdout:Get:2 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2024-09-13T00:05:14.663 INFO:teuthology.orchestra.run.smithi077.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2024-09-13T00:05:14.671 INFO:teuthology.orchestra.run.smithi088.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [128 kB] 2024-09-13T00:05:14.727 INFO:teuthology.orchestra.run.smithi077.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2024-09-13T00:05:14.736 INFO:teuthology.orchestra.run.smithi088.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [127 kB] 2024-09-13T00:05:14.762 INFO:teuthology.orchestra.run.smithi088.stdout:Get:4 http://security.ubuntu.com/ubuntu jammy-security InRelease [129 kB] 2024-09-13T00:05:14.901 INFO:teuthology.orchestra.run.smithi077.stdout:Get:5 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [533 kB] 2024-09-13T00:05:14.969 INFO:teuthology.orchestra.run.smithi077.stdout:Get:6 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,773 kB] 2024-09-13T00:05:14.979 INFO:teuthology.orchestra.run.smithi077.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2,022 kB] 2024-09-13T00:05:15.006 INFO:teuthology.orchestra.run.smithi077.stdout:Get:8 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [291 kB] 2024-09-13T00:05:15.008 INFO:teuthology.orchestra.run.smithi088.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [696 kB] 2024-09-13T00:05:15.016 INFO:teuthology.orchestra.run.smithi077.stdout:Get:9 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [13.3 kB] 2024-09-13T00:05:15.016 INFO:teuthology.orchestra.run.smithi077.stdout:Get:10 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,330 kB] 2024-09-13T00:05:15.033 INFO:teuthology.orchestra.run.smithi077.stdout:Get:11 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [37.7 kB] 2024-09-13T00:05:15.039 INFO:teuthology.orchestra.run.smithi077.stdout:Get:12 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [401 kB] 2024-09-13T00:05:15.042 INFO:teuthology.orchestra.run.smithi077.stdout:Get:13 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [584 B] 2024-09-13T00:05:15.042 INFO:teuthology.orchestra.run.smithi077.stdout:Get:14 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [901 kB] 2024-09-13T00:05:15.048 INFO:teuthology.orchestra.run.smithi077.stdout:Get:15 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [626 kB] 2024-09-13T00:05:15.051 INFO:teuthology.orchestra.run.smithi077.stdout:Get:16 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [176 kB] 2024-09-13T00:05:15.053 INFO:teuthology.orchestra.run.smithi077.stdout:Get:17 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [19.2 kB] 2024-09-13T00:05:15.053 INFO:teuthology.orchestra.run.smithi077.stdout:Get:18 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-09-13T00:05:15.053 INFO:teuthology.orchestra.run.smithi077.stdout:Get:19 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB] 2024-09-13T00:05:15.070 INFO:teuthology.orchestra.run.smithi088.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [2,022 kB] 2024-09-13T00:05:15.086 INFO:teuthology.orchestra.run.smithi077.stdout:Get:20 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B] 2024-09-13T00:05:15.086 INFO:teuthology.orchestra.run.smithi077.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [696 kB] 2024-09-13T00:05:15.086 INFO:teuthology.orchestra.run.smithi077.stdout:Get:22 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [228 B] 2024-09-13T00:05:15.093 INFO:teuthology.orchestra.run.smithi077.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [352 kB] 2024-09-13T00:05:15.097 INFO:teuthology.orchestra.run.smithi077.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [17.8 kB] 2024-09-13T00:05:15.098 INFO:teuthology.orchestra.run.smithi077.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,437 kB] 2024-09-13T00:05:15.103 INFO:teuthology.orchestra.run.smithi088.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [352 kB] 2024-09-13T00:05:15.106 INFO:teuthology.orchestra.run.smithi088.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [17.8 kB] 2024-09-13T00:05:15.107 INFO:teuthology.orchestra.run.smithi088.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [39.3 kB] 2024-09-13T00:05:15.107 INFO:teuthology.orchestra.run.smithi088.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [2,437 kB] 2024-09-13T00:05:15.122 INFO:teuthology.orchestra.run.smithi077.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [39.3 kB] 2024-09-13T00:05:15.123 INFO:teuthology.orchestra.run.smithi077.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [419 kB] 2024-09-13T00:05:15.126 INFO:teuthology.orchestra.run.smithi077.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [616 B] 2024-09-13T00:05:15.126 INFO:teuthology.orchestra.run.smithi077.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [732 kB] 2024-09-13T00:05:15.131 INFO:teuthology.orchestra.run.smithi077.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,124 kB] 2024-09-13T00:05:15.135 INFO:teuthology.orchestra.run.smithi088.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [419 kB] 2024-09-13T00:05:15.137 INFO:teuthology.orchestra.run.smithi088.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [616 B] 2024-09-13T00:05:15.137 INFO:teuthology.orchestra.run.smithi088.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,124 kB] 2024-09-13T00:05:15.138 INFO:teuthology.orchestra.run.smithi077.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [261 kB] 2024-09-13T00:05:15.139 INFO:teuthology.orchestra.run.smithi077.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [26.2 kB] 2024-09-13T00:05:15.140 INFO:teuthology.orchestra.run.smithi077.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [43.3 kB] 2024-09-13T00:05:15.140 INFO:teuthology.orchestra.run.smithi077.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,752 B] 2024-09-13T00:05:15.148 INFO:teuthology.orchestra.run.smithi077.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.8 kB] 2024-09-13T00:05:15.148 INFO:teuthology.orchestra.run.smithi077.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [444 B] 2024-09-13T00:05:15.148 INFO:teuthology.orchestra.run.smithi077.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.8 kB] 2024-09-13T00:05:15.148 INFO:teuthology.orchestra.run.smithi077.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.9 kB] 2024-09-13T00:05:15.149 INFO:teuthology.orchestra.run.smithi088.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [732 kB] 2024-09-13T00:05:15.156 INFO:teuthology.orchestra.run.smithi088.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [261 kB] 2024-09-13T00:05:15.158 INFO:teuthology.orchestra.run.smithi077.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.1 kB] 2024-09-13T00:05:15.158 INFO:teuthology.orchestra.run.smithi077.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-09-13T00:05:15.158 INFO:teuthology.orchestra.run.smithi077.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [28.8 kB] 2024-09-13T00:05:15.159 INFO:teuthology.orchestra.run.smithi077.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [17.3 kB] 2024-09-13T00:05:15.159 INFO:teuthology.orchestra.run.smithi077.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-09-13T00:05:15.159 INFO:teuthology.orchestra.run.smithi088.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [26.2 kB] 2024-09-13T00:05:15.160 INFO:teuthology.orchestra.run.smithi088.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,752 B] 2024-09-13T00:05:15.164 INFO:teuthology.orchestra.run.smithi077.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [672 B] 2024-09-13T00:05:15.188 INFO:teuthology.orchestra.run.smithi088.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [43.3 kB] 2024-09-13T00:05:15.189 INFO:teuthology.orchestra.run.smithi088.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.8 kB] 2024-09-13T00:05:15.189 INFO:teuthology.orchestra.run.smithi088.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [444 B] 2024-09-13T00:05:15.189 INFO:teuthology.orchestra.run.smithi088.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.9 kB] 2024-09-13T00:05:15.190 INFO:teuthology.orchestra.run.smithi088.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.8 kB] 2024-09-13T00:05:15.191 INFO:teuthology.orchestra.run.smithi088.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.1 kB] 2024-09-13T00:05:15.191 INFO:teuthology.orchestra.run.smithi088.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-09-13T00:05:15.191 INFO:teuthology.orchestra.run.smithi088.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [28.8 kB] 2024-09-13T00:05:15.206 INFO:teuthology.orchestra.run.smithi088.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [17.3 kB] 2024-09-13T00:05:15.207 INFO:teuthology.orchestra.run.smithi088.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-09-13T00:05:15.207 INFO:teuthology.orchestra.run.smithi088.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [672 B] 2024-09-13T00:05:15.217 INFO:teuthology.orchestra.run.smithi088.stdout:Get:29 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [533 kB] 2024-09-13T00:05:15.499 INFO:teuthology.orchestra.run.smithi088.stdout:Get:30 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,773 kB] 2024-09-13T00:05:15.671 INFO:teuthology.orchestra.run.smithi088.stdout:Get:31 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [291 kB] 2024-09-13T00:05:15.683 INFO:teuthology.orchestra.run.smithi088.stdout:Get:32 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [13.3 kB] 2024-09-13T00:05:15.684 INFO:teuthology.orchestra.run.smithi088.stdout:Get:33 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [2,330 kB] 2024-09-13T00:05:15.769 INFO:teuthology.orchestra.run.smithi088.stdout:Get:34 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [37.7 kB] 2024-09-13T00:05:15.769 INFO:teuthology.orchestra.run.smithi088.stdout:Get:35 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [401 kB] 2024-09-13T00:05:15.778 INFO:teuthology.orchestra.run.smithi088.stdout:Get:36 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [584 B] 2024-09-13T00:05:15.778 INFO:teuthology.orchestra.run.smithi088.stdout:Get:37 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [626 kB] 2024-09-13T00:05:15.791 INFO:teuthology.orchestra.run.smithi088.stdout:Get:38 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [901 kB] 2024-09-13T00:05:15.830 INFO:teuthology.orchestra.run.smithi088.stdout:Get:39 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [176 kB] 2024-09-13T00:05:15.833 INFO:teuthology.orchestra.run.smithi088.stdout:Get:40 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [19.2 kB] 2024-09-13T00:05:15.836 INFO:teuthology.orchestra.run.smithi088.stdout:Get:41 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-09-13T00:05:15.836 INFO:teuthology.orchestra.run.smithi088.stdout:Get:42 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB] 2024-09-13T00:05:15.917 INFO:teuthology.orchestra.run.smithi088.stdout:Get:43 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B] 2024-09-13T00:05:15.917 INFO:teuthology.orchestra.run.smithi088.stdout:Get:44 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [228 B] 2024-09-13T00:05:21.273 INFO:teuthology.orchestra.run.smithi077.stdout:Fetched 15.9 MB in 2s (7,270 kB/s) 2024-09-13T00:05:21.429 INFO:teuthology.orchestra.run.smithi088.stdout:Fetched 15.9 MB in 2s (6,814 kB/s) 2024-09-13T00:05:22.791 INFO:teuthology.orchestra.run.smithi077.stdout:Reading package lists... 2024-09-13T00:05:22.811 DEBUG:teuthology.orchestra.run.smithi077:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-13T00:05:22.858 INFO:teuthology.orchestra.run.smithi088.stdout:Reading package lists... 2024-09-13T00:05:22.874 INFO:teuthology.orchestra.run.smithi077.stdout:Reading package lists... 2024-09-13T00:05:22.876 DEBUG:teuthology.orchestra.run.smithi088:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-13T00:05:22.931 INFO:teuthology.orchestra.run.smithi088.stdout:Reading package lists... 2024-09-13T00:05:23.063 INFO:teuthology.orchestra.run.smithi077.stdout:Building dependency tree... 2024-09-13T00:05:23.063 INFO:teuthology.orchestra.run.smithi077.stdout:Reading state information... 2024-09-13T00:05:23.118 INFO:teuthology.orchestra.run.smithi088.stdout:Building dependency tree... 2024-09-13T00:05:23.118 INFO:teuthology.orchestra.run.smithi088.stdout:Reading state information... 2024-09-13T00:05:23.230 INFO:teuthology.orchestra.run.smithi077.stdout:The following packages were automatically installed and are no longer required: 2024-09-13T00:05:23.230 INFO:teuthology.orchestra.run.smithi077.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-13T00:05:23.231 INFO:teuthology.orchestra.run.smithi077.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-13T00:05:23.231 INFO:teuthology.orchestra.run.smithi077.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-13T00:05:23.231 INFO:teuthology.orchestra.run.smithi077.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-09-13T00:05:23.231 INFO:teuthology.orchestra.run.smithi077.stdout: python2.7-minimal 2024-09-13T00:05:23.231 INFO:teuthology.orchestra.run.smithi077.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-13T00:05:23.232 INFO:teuthology.orchestra.run.smithi077.stdout:The following additional packages will be installed: 2024-09-13T00:05:23.233 INFO:teuthology.orchestra.run.smithi077.stdout: linux-generic linux-headers-5.15.0-121 linux-headers-5.15.0-121-generic 2024-09-13T00:05:23.233 INFO:teuthology.orchestra.run.smithi077.stdout: linux-headers-generic linux-image-5.15.0-121-generic 2024-09-13T00:05:23.233 INFO:teuthology.orchestra.run.smithi077.stdout: linux-modules-5.15.0-121-generic linux-modules-extra-5.15.0-121-generic 2024-09-13T00:05:23.234 INFO:teuthology.orchestra.run.smithi077.stdout:Suggested packages: 2024-09-13T00:05:23.234 INFO:teuthology.orchestra.run.smithi077.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-09-13T00:05:23.234 INFO:teuthology.orchestra.run.smithi077.stdout:Recommended packages: 2024-09-13T00:05:23.234 INFO:teuthology.orchestra.run.smithi077.stdout: thermald 2024-09-13T00:05:23.273 INFO:teuthology.orchestra.run.smithi077.stdout:The following NEW packages will be installed: 2024-09-13T00:05:23.273 INFO:teuthology.orchestra.run.smithi077.stdout: linux-headers-5.15.0-121 linux-headers-5.15.0-121-generic 2024-09-13T00:05:23.273 INFO:teuthology.orchestra.run.smithi077.stdout: linux-image-5.15.0-121-generic linux-modules-5.15.0-121-generic 2024-09-13T00:05:23.274 INFO:teuthology.orchestra.run.smithi077.stdout: linux-modules-extra-5.15.0-121-generic 2024-09-13T00:05:23.275 INFO:teuthology.orchestra.run.smithi077.stdout:The following packages will be upgraded: 2024-09-13T00:05:23.275 INFO:teuthology.orchestra.run.smithi077.stdout: linux-generic linux-headers-generic linux-image-generic 2024-09-13T00:05:23.288 INFO:teuthology.orchestra.run.smithi088.stdout:The following packages were automatically installed and are no longer required: 2024-09-13T00:05:23.288 INFO:teuthology.orchestra.run.smithi088.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-13T00:05:23.288 INFO:teuthology.orchestra.run.smithi088.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-13T00:05:23.288 INFO:teuthology.orchestra.run.smithi088.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-13T00:05:23.288 INFO:teuthology.orchestra.run.smithi088.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-09-13T00:05:23.289 INFO:teuthology.orchestra.run.smithi088.stdout: python2.7-minimal 2024-09-13T00:05:23.289 INFO:teuthology.orchestra.run.smithi088.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-13T00:05:23.289 INFO:teuthology.orchestra.run.smithi088.stdout:The following additional packages will be installed: 2024-09-13T00:05:23.290 INFO:teuthology.orchestra.run.smithi088.stdout: linux-generic linux-headers-5.15.0-121 linux-headers-5.15.0-121-generic 2024-09-13T00:05:23.290 INFO:teuthology.orchestra.run.smithi088.stdout: linux-headers-generic linux-image-5.15.0-121-generic 2024-09-13T00:05:23.290 INFO:teuthology.orchestra.run.smithi088.stdout: linux-modules-5.15.0-121-generic linux-modules-extra-5.15.0-121-generic 2024-09-13T00:05:23.291 INFO:teuthology.orchestra.run.smithi088.stdout:Suggested packages: 2024-09-13T00:05:23.291 INFO:teuthology.orchestra.run.smithi088.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-09-13T00:05:23.292 INFO:teuthology.orchestra.run.smithi088.stdout:Recommended packages: 2024-09-13T00:05:23.292 INFO:teuthology.orchestra.run.smithi088.stdout: thermald 2024-09-13T00:05:23.338 INFO:teuthology.orchestra.run.smithi088.stdout:The following NEW packages will be installed: 2024-09-13T00:05:23.338 INFO:teuthology.orchestra.run.smithi088.stdout: linux-headers-5.15.0-121 linux-headers-5.15.0-121-generic 2024-09-13T00:05:23.338 INFO:teuthology.orchestra.run.smithi088.stdout: linux-image-5.15.0-121-generic linux-modules-5.15.0-121-generic 2024-09-13T00:05:23.339 INFO:teuthology.orchestra.run.smithi088.stdout: linux-modules-extra-5.15.0-121-generic 2024-09-13T00:05:23.340 INFO:teuthology.orchestra.run.smithi088.stdout:The following packages will be upgraded: 2024-09-13T00:05:23.340 INFO:teuthology.orchestra.run.smithi088.stdout: linux-generic linux-headers-generic linux-image-generic 2024-09-13T00:05:23.454 INFO:teuthology.orchestra.run.smithi077.stdout:3 upgraded, 5 newly installed, 0 to remove and 344 not upgraded. 2024-09-13T00:05:23.454 INFO:teuthology.orchestra.run.smithi077.stdout:Need to get 113 MB of archives. 2024-09-13T00:05:23.454 INFO:teuthology.orchestra.run.smithi077.stdout:After this operation, 584 MB of additional disk space will be used. 2024-09-13T00:05:23.454 INFO:teuthology.orchestra.run.smithi077.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-121-generic amd64 5.15.0-121.131 [22.7 MB] 2024-09-13T00:05:23.520 INFO:teuthology.orchestra.run.smithi088.stdout:3 upgraded, 5 newly installed, 0 to remove and 344 not upgraded. 2024-09-13T00:05:23.520 INFO:teuthology.orchestra.run.smithi088.stdout:Need to get 113 MB of archives. 2024-09-13T00:05:23.521 INFO:teuthology.orchestra.run.smithi088.stdout:After this operation, 584 MB of additional disk space will be used. 2024-09-13T00:05:23.521 INFO:teuthology.orchestra.run.smithi088.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-121-generic amd64 5.15.0-121.131 [22.7 MB] 2024-09-13T00:05:24.927 INFO:teuthology.orchestra.run.smithi077.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-121-generic amd64 5.15.0-121.131 [11.5 MB] 2024-09-13T00:05:25.065 INFO:teuthology.orchestra.run.smithi088.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-121-generic amd64 5.15.0-121.131 [11.5 MB] 2024-09-13T00:05:25.277 INFO:teuthology.orchestra.run.smithi077.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-121-generic amd64 5.15.0-121.131 [63.9 MB] 2024-09-13T00:05:25.417 INFO:teuthology.orchestra.run.smithi088.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-121-generic amd64 5.15.0-121.131 [63.9 MB] 2024-09-13T00:05:27.460 INFO:teuthology.orchestra.run.smithi077.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.121.121 [1,694 B] 2024-09-13T00:05:27.460 INFO:teuthology.orchestra.run.smithi077.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.121.121 [2,464 B] 2024-09-13T00:05:27.461 INFO:teuthology.orchestra.run.smithi077.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-121 all 5.15.0-121.131 [12.3 MB] 2024-09-13T00:05:27.648 INFO:teuthology.orchestra.run.smithi088.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.121.121 [1,694 B] 2024-09-13T00:05:27.648 INFO:teuthology.orchestra.run.smithi088.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.121.121 [2,464 B] 2024-09-13T00:05:27.648 INFO:teuthology.orchestra.run.smithi088.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-121 all 5.15.0-121.131 [12.3 MB] 2024-09-13T00:05:27.814 INFO:teuthology.orchestra.run.smithi077.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-121-generic amd64 5.15.0-121.131 [2,827 kB] 2024-09-13T00:05:27.897 INFO:teuthology.orchestra.run.smithi077.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.121.121 [2,316 B] 2024-09-13T00:05:28.310 INFO:teuthology.orchestra.run.smithi077.stdout:Fetched 113 MB in 5s (24.5 MB/s) 2024-09-13T00:05:28.379 INFO:teuthology.orchestra.run.smithi088.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-121-generic amd64 5.15.0-121.131 [2,827 kB] 2024-09-13T00:05:28.503 INFO:teuthology.orchestra.run.smithi088.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.121.121 [2,316 B] 2024-09-13T00:05:28.514 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package linux-modules-5.15.0-121-generic. 2024-09-13T00:05:28.929 INFO:teuthology.orchestra.run.smithi088.stdout:Fetched 113 MB in 5s (22.0 MB/s) 2024-09-13T00:05:29.117 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package linux-modules-5.15.0-121-generic. 2024-09-13T00:05:30.538 INFO:teuthology.orchestra.run.smithi077.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-09-13T00:05:30.542 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../0-linux-modules-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-13T00:05:30.637 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking linux-modules-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:31.130 INFO:teuthology.orchestra.run.smithi088.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-09-13T00:05:31.134 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../0-linux-modules-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-13T00:05:31.282 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking linux-modules-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:33.081 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package linux-image-5.15.0-121-generic. 2024-09-13T00:05:33.102 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../1-linux-image-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-13T00:05:33.195 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:33.659 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-121-generic. 2024-09-13T00:05:33.680 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-13T00:05:33.717 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking linux-modules-extra-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:33.750 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package linux-image-5.15.0-121-generic. 2024-09-13T00:05:33.771 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../1-linux-image-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-13T00:05:33.865 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:34.328 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-121-generic. 2024-09-13T00:05:34.349 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-13T00:05:34.386 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking linux-modules-extra-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:40.265 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../3-linux-generic_5.15.0.121.121_amd64.deb ... 2024-09-13T00:05:40.373 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking linux-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-13T00:05:40.709 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../3-linux-generic_5.15.0.121.121_amd64.deb ... 2024-09-13T00:05:40.740 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.121.121_amd64.deb ... 2024-09-13T00:05:40.836 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking linux-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-13T00:05:40.868 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking linux-image-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-13T00:05:41.186 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.121.121_amd64.deb ... 2024-09-13T00:05:41.196 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package linux-headers-5.15.0-121. 2024-09-13T00:05:41.218 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../5-linux-headers-5.15.0-121_5.15.0-121.131_all.deb ... 2024-09-13T00:05:41.254 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking linux-headers-5.15.0-121 (5.15.0-121.131) ... 2024-09-13T00:05:41.298 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking linux-image-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-13T00:05:41.559 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package linux-headers-5.15.0-121. 2024-09-13T00:05:41.572 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../5-linux-headers-5.15.0-121_5.15.0-121.131_all.deb ... 2024-09-13T00:05:41.617 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking linux-headers-5.15.0-121 (5.15.0-121.131) ... 2024-09-13T00:05:47.283 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package linux-headers-5.15.0-121-generic. 2024-09-13T00:05:47.320 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../6-linux-headers-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-13T00:05:47.358 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking linux-headers-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:47.373 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package linux-headers-5.15.0-121-generic. 2024-09-13T00:05:47.411 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../6-linux-headers-5.15.0-121-generic_5.15.0-121.131_amd64.deb ... 2024-09-13T00:05:47.448 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking linux-headers-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:50.301 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.121.121_amd64.deb ... 2024-09-13T00:05:51.049 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.121.121_amd64.deb ... 2024-09-13T00:05:51.092 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking linux-headers-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-13T00:05:51.172 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking linux-headers-generic (5.15.0.121.121) over (5.15.0.56.54) ... 2024-09-13T00:05:51.434 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up linux-headers-5.15.0-121 (5.15.0-121.131) ... 2024-09-13T00:05:51.547 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up linux-headers-5.15.0-121 (5.15.0-121.131) ... 2024-09-13T00:05:51.564 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up linux-headers-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:51.669 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up linux-headers-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:51.690 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up linux-headers-generic (5.15.0.121.121) ... 2024-09-13T00:05:51.795 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up linux-headers-generic (5.15.0.121.121) ... 2024-09-13T00:05:51.843 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:51.948 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:53.140 INFO:teuthology.orchestra.run.smithi088.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-09-13T00:05:53.140 INFO:teuthology.orchestra.run.smithi088.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-09-13T00:05:53.140 INFO:teuthology.orchestra.run.smithi088.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-121-generic 2024-09-13T00:05:53.140 INFO:teuthology.orchestra.run.smithi088.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-121-generic 2024-09-13T00:05:53.262 INFO:teuthology.orchestra.run.smithi077.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-09-13T00:05:53.262 INFO:teuthology.orchestra.run.smithi077.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-09-13T00:05:53.262 INFO:teuthology.orchestra.run.smithi077.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-121-generic 2024-09-13T00:05:53.262 INFO:teuthology.orchestra.run.smithi077.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-121-generic 2024-09-13T00:05:53.287 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up linux-modules-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:53.435 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up linux-modules-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:54.663 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up linux-modules-extra-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:54.877 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up linux-modules-extra-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:56.848 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up linux-image-generic (5.15.0.121.121) ... 2024-09-13T00:05:56.974 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up linux-generic (5.15.0.121.121) ... 2024-09-13T00:05:57.021 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up linux-image-generic (5.15.0.121.121) ... 2024-09-13T00:05:57.100 INFO:teuthology.orchestra.run.smithi088.stdout:Processing triggers for linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:57.130 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up linux-generic (5.15.0.121.121) ... 2024-09-13T00:05:57.161 INFO:teuthology.orchestra.run.smithi088.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-09-13T00:05:57.161 INFO:teuthology.orchestra.run.smithi088.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-121-generic 2024-09-13T00:05:57.232 INFO:teuthology.orchestra.run.smithi077.stdout:Processing triggers for linux-image-5.15.0-121-generic (5.15.0-121.131) ... 2024-09-13T00:05:57.296 INFO:teuthology.orchestra.run.smithi077.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-09-13T00:05:57.296 INFO:teuthology.orchestra.run.smithi077.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-121-generic 2024-09-13T00:06:11.087 INFO:teuthology.orchestra.run.smithi088.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-09-13T00:06:11.087 INFO:teuthology.orchestra.run.smithi088.stdout:Sourcing file `/etc/default/grub' 2024-09-13T00:06:11.105 INFO:teuthology.orchestra.run.smithi088.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-13T00:06:11.133 INFO:teuthology.orchestra.run.smithi088.stdout:Generating grub configuration file ... 2024-09-13T00:06:11.456 INFO:teuthology.orchestra.run.smithi088.stdout:Found linux image: /boot/vmlinuz-5.15.0-121-generic 2024-09-13T00:06:11.483 INFO:teuthology.orchestra.run.smithi088.stdout:Found initrd image: /boot/initrd.img-5.15.0-121-generic 2024-09-13T00:06:11.485 INFO:teuthology.orchestra.run.smithi077.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-09-13T00:06:11.485 INFO:teuthology.orchestra.run.smithi077.stdout:Sourcing file `/etc/default/grub' 2024-09-13T00:06:11.490 INFO:teuthology.orchestra.run.smithi077.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-13T00:06:11.522 INFO:teuthology.orchestra.run.smithi077.stdout:Generating grub configuration file ... 2024-09-13T00:06:11.815 INFO:teuthology.orchestra.run.smithi077.stdout:Found linux image: /boot/vmlinuz-5.15.0-121-generic 2024-09-13T00:06:11.837 INFO:teuthology.orchestra.run.smithi077.stdout:Found initrd image: /boot/initrd.img-5.15.0-121-generic 2024-09-13T00:06:11.925 INFO:teuthology.orchestra.run.smithi088.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-13T00:06:11.931 INFO:teuthology.orchestra.run.smithi088.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-13T00:06:12.006 INFO:teuthology.orchestra.run.smithi088.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-13T00:06:12.012 INFO:teuthology.orchestra.run.smithi088.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-13T00:06:12.141 INFO:teuthology.orchestra.run.smithi077.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-13T00:06:12.146 INFO:teuthology.orchestra.run.smithi077.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-13T00:06:12.175 INFO:teuthology.orchestra.run.smithi088.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-13T00:06:12.176 INFO:teuthology.orchestra.run.smithi088.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-09-13T00:06:12.176 INFO:teuthology.orchestra.run.smithi088.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-13T00:06:12.216 INFO:teuthology.orchestra.run.smithi077.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-13T00:06:12.218 INFO:teuthology.orchestra.run.smithi088.stdout:done 2024-09-13T00:06:12.221 INFO:teuthology.orchestra.run.smithi077.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-13T00:06:12.383 INFO:teuthology.orchestra.run.smithi077.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-13T00:06:12.383 INFO:teuthology.orchestra.run.smithi077.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-09-13T00:06:12.384 INFO:teuthology.orchestra.run.smithi077.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-13T00:06:12.422 INFO:teuthology.orchestra.run.smithi077.stdout:done 2024-09-13T00:06:12.746 DEBUG:teuthology.orchestra.run.smithi088:> dpkg -s linux-image-generic 2024-09-13T00:06:12.772 INFO:teuthology.orchestra.run.smithi088.stdout:Package: linux-image-generic 2024-09-13T00:06:12.773 INFO:teuthology.orchestra.run.smithi088.stdout:Status: install ok installed 2024-09-13T00:06:12.773 INFO:teuthology.orchestra.run.smithi088.stdout:Priority: optional 2024-09-13T00:06:12.773 INFO:teuthology.orchestra.run.smithi088.stdout:Section: kernel 2024-09-13T00:06:12.773 INFO:teuthology.orchestra.run.smithi088.stdout:Installed-Size: 21 2024-09-13T00:06:12.773 INFO:teuthology.orchestra.run.smithi088.stdout:Maintainer: Ubuntu Kernel Team 2024-09-13T00:06:12.773 INFO:teuthology.orchestra.run.smithi088.stdout:Architecture: amd64 2024-09-13T00:06:12.773 INFO:teuthology.orchestra.run.smithi088.stdout:Source: linux-meta 2024-09-13T00:06:12.773 INFO:teuthology.orchestra.run.smithi088.stdout:Version: 5.15.0.121.121 2024-09-13T00:06:12.773 INFO:teuthology.orchestra.run.smithi088.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-121), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-13T00:06:12.773 INFO:teuthology.orchestra.run.smithi088.stdout:Depends: linux-image-5.15.0-121-generic, linux-modules-extra-5.15.0-121-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-13T00:06:12.773 INFO:teuthology.orchestra.run.smithi088.stdout:Recommends: thermald 2024-09-13T00:06:12.774 INFO:teuthology.orchestra.run.smithi088.stdout:Description: Generic Linux kernel image 2024-09-13T00:06:12.774 INFO:teuthology.orchestra.run.smithi088.stdout: This package will always depend on the latest generic kernel image 2024-09-13T00:06:12.774 INFO:teuthology.orchestra.run.smithi088.stdout: available. 2024-09-13T00:06:12.774 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-121-generic 2024-09-13T00:06:12.774 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T00:06:12.775 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T00:06:12.775 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-13T00:06:12.775 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi088.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T00:06:12.775 DEBUG:teuthology.orchestra.run.smithi088:> sudo apt-get clean 2024-09-13T00:06:12.858 DEBUG:teuthology.orchestra.run.smithi077:> dpkg -s linux-image-generic 2024-09-13T00:06:12.879 INFO:teuthology.orchestra.run.smithi077.stdout:Package: linux-image-generic 2024-09-13T00:06:12.880 INFO:teuthology.orchestra.run.smithi077.stdout:Status: install ok installed 2024-09-13T00:06:12.880 INFO:teuthology.orchestra.run.smithi077.stdout:Priority: optional 2024-09-13T00:06:12.880 INFO:teuthology.orchestra.run.smithi077.stdout:Section: kernel 2024-09-13T00:06:12.880 INFO:teuthology.orchestra.run.smithi077.stdout:Installed-Size: 21 2024-09-13T00:06:12.880 INFO:teuthology.orchestra.run.smithi077.stdout:Maintainer: Ubuntu Kernel Team 2024-09-13T00:06:12.880 INFO:teuthology.orchestra.run.smithi077.stdout:Architecture: amd64 2024-09-13T00:06:12.880 INFO:teuthology.orchestra.run.smithi077.stdout:Source: linux-meta 2024-09-13T00:06:12.880 INFO:teuthology.orchestra.run.smithi077.stdout:Version: 5.15.0.121.121 2024-09-13T00:06:12.880 INFO:teuthology.orchestra.run.smithi077.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-121), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-13T00:06:12.880 INFO:teuthology.orchestra.run.smithi077.stdout:Depends: linux-image-5.15.0-121-generic, linux-modules-extra-5.15.0-121-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-13T00:06:12.880 INFO:teuthology.orchestra.run.smithi077.stdout:Recommends: thermald 2024-09-13T00:06:12.880 INFO:teuthology.orchestra.run.smithi077.stdout:Description: Generic Linux kernel image 2024-09-13T00:06:12.881 INFO:teuthology.orchestra.run.smithi077.stdout: This package will always depend on the latest generic kernel image 2024-09-13T00:06:12.881 INFO:teuthology.orchestra.run.smithi077.stdout: available. 2024-09-13T00:06:12.881 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-121-generic 2024-09-13T00:06:12.881 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-13T00:06:12.881 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-13T00:06:12.881 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-13T00:06:12.881 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi077.front.sepia.ceph.com, path=None, version=distro) 2024-09-13T00:06:12.881 DEBUG:teuthology.orchestra.run.smithi077:> sudo apt-get clean 2024-09-13T00:06:12.962 DEBUG:teuthology.orchestra.run.smithi088:> sudo apt-get update 2024-09-13T00:06:13.064 DEBUG:teuthology.orchestra.run.smithi077:> sudo apt-get update 2024-09-13T00:06:13.081 INFO:teuthology.orchestra.run.smithi088.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-13T00:06:13.091 INFO:teuthology.orchestra.run.smithi088.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-13T00:06:13.109 INFO:teuthology.orchestra.run.smithi088.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-13T00:06:13.204 INFO:teuthology.orchestra.run.smithi088.stdout:Hit:4 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-13T00:06:13.322 INFO:teuthology.orchestra.run.smithi077.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-13T00:06:13.322 INFO:teuthology.orchestra.run.smithi077.stdout:Hit:2 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-13T00:06:13.393 INFO:teuthology.orchestra.run.smithi077.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-13T00:06:13.474 INFO:teuthology.orchestra.run.smithi077.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-13T00:06:14.513 INFO:teuthology.orchestra.run.smithi088.stdout:Reading package lists... 2024-09-13T00:06:14.532 DEBUG:teuthology.orchestra.run.smithi088:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-13T00:06:14.592 INFO:teuthology.orchestra.run.smithi088.stdout:Reading package lists... 2024-09-13T00:06:14.751 INFO:teuthology.orchestra.run.smithi077.stdout:Reading package lists... 2024-09-13T00:06:14.768 DEBUG:teuthology.orchestra.run.smithi077:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-09-13T00:06:14.802 INFO:teuthology.orchestra.run.smithi088.stdout:Building dependency tree... 2024-09-13T00:06:14.803 INFO:teuthology.orchestra.run.smithi088.stdout:Reading state information... 2024-09-13T00:06:14.834 INFO:teuthology.orchestra.run.smithi077.stdout:Reading package lists... 2024-09-13T00:06:14.985 INFO:teuthology.orchestra.run.smithi088.stdout:linux-image-generic is already the newest version (5.15.0.121.121). 2024-09-13T00:06:14.985 INFO:teuthology.orchestra.run.smithi088.stdout:The following packages were automatically installed and are no longer required: 2024-09-13T00:06:14.985 INFO:teuthology.orchestra.run.smithi088.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-13T00:06:14.985 INFO:teuthology.orchestra.run.smithi088.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-13T00:06:14.985 INFO:teuthology.orchestra.run.smithi088.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-13T00:06:14.986 INFO:teuthology.orchestra.run.smithi088.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-13T00:06:14.986 INFO:teuthology.orchestra.run.smithi088.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-13T00:06:14.986 INFO:teuthology.orchestra.run.smithi088.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-13T00:06:15.024 INFO:teuthology.orchestra.run.smithi088.stdout:0 upgraded, 0 newly installed, 0 to remove and 344 not upgraded. 2024-09-13T00:06:15.026 DEBUG:teuthology.orchestra.run.smithi088:> dpkg -s linux-image-generic 2024-09-13T00:06:15.041 INFO:teuthology.orchestra.run.smithi088.stdout:Package: linux-image-generic 2024-09-13T00:06:15.041 INFO:teuthology.orchestra.run.smithi088.stdout:Status: install ok installed 2024-09-13T00:06:15.041 INFO:teuthology.orchestra.run.smithi088.stdout:Priority: optional 2024-09-13T00:06:15.041 INFO:teuthology.orchestra.run.smithi088.stdout:Section: kernel 2024-09-13T00:06:15.042 INFO:teuthology.orchestra.run.smithi088.stdout:Installed-Size: 21 2024-09-13T00:06:15.042 INFO:teuthology.orchestra.run.smithi088.stdout:Maintainer: Ubuntu Kernel Team 2024-09-13T00:06:15.042 INFO:teuthology.orchestra.run.smithi088.stdout:Architecture: amd64 2024-09-13T00:06:15.042 INFO:teuthology.orchestra.run.smithi088.stdout:Source: linux-meta 2024-09-13T00:06:15.042 INFO:teuthology.orchestra.run.smithi088.stdout:Version: 5.15.0.121.121 2024-09-13T00:06:15.042 INFO:teuthology.orchestra.run.smithi088.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-121), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-13T00:06:15.042 INFO:teuthology.orchestra.run.smithi088.stdout:Depends: linux-image-5.15.0-121-generic, linux-modules-extra-5.15.0-121-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-13T00:06:15.042 INFO:teuthology.orchestra.run.smithi088.stdout:Recommends: thermald 2024-09-13T00:06:15.042 INFO:teuthology.orchestra.run.smithi088.stdout:Description: Generic Linux kernel image 2024-09-13T00:06:15.042 INFO:teuthology.orchestra.run.smithi088.stdout: This package will always depend on the latest generic kernel image 2024-09-13T00:06:15.042 INFO:teuthology.orchestra.run.smithi088.stdout: available. 2024-09-13T00:06:15.043 DEBUG:teuthology.orchestra.run.smithi088:> mktemp 2024-09-13T00:06:15.046 INFO:teuthology.orchestra.run.smithi077.stdout:Building dependency tree... 2024-09-13T00:06:15.047 INFO:teuthology.orchestra.run.smithi077.stdout:Reading state information... 2024-09-13T00:06:15.088 INFO:teuthology.orchestra.run.smithi088.stdout:/tmp/tmp.nokA9iacpG 2024-09-13T00:06:15.088 DEBUG:teuthology.orchestra.run.smithi088:> sudo cp /boot/grub/grub.cfg /tmp/tmp.nokA9iacpG 2024-09-13T00:06:15.143 DEBUG:teuthology.orchestra.run.smithi088:> sudo chmod 0666 /tmp/tmp.nokA9iacpG 2024-09-13T00:06:15.240 INFO:teuthology.orchestra.run.smithi077.stdout:linux-image-generic is already the newest version (5.15.0.121.121). 2024-09-13T00:06:15.240 INFO:teuthology.orchestra.run.smithi077.stdout:The following packages were automatically installed and are no longer required: 2024-09-13T00:06:15.240 INFO:teuthology.orchestra.run.smithi077.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-13T00:06:15.240 INFO:teuthology.orchestra.run.smithi077.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-13T00:06:15.240 INFO:teuthology.orchestra.run.smithi077.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-13T00:06:15.241 INFO:teuthology.orchestra.run.smithi077.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-13T00:06:15.241 INFO:teuthology.orchestra.run.smithi077.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-13T00:06:15.241 INFO:teuthology.orchestra.run.smithi077.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-13T00:06:15.276 DEBUG:teuthology.orchestra.remote:smithi088:/tmp/tmp.nokA9iacpG is 10KB 2024-09-13T00:06:15.283 INFO:teuthology.orchestra.run.smithi077.stdout:0 upgraded, 0 newly installed, 0 to remove and 344 not upgraded. 2024-09-13T00:06:15.285 DEBUG:teuthology.orchestra.run.smithi077:> dpkg -s linux-image-generic 2024-09-13T00:06:15.290 DEBUG:teuthology.orchestra.run.smithi088:> rm -fr /tmp/tmp.nokA9iacpG 2024-09-13T00:06:15.296 DEBUG:teuthology.orchestra.run.smithi088:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-09-13T00:06:15.301 INFO:teuthology.orchestra.run.smithi077.stdout:Package: linux-image-generic 2024-09-13T00:06:15.301 INFO:teuthology.orchestra.run.smithi077.stdout:Status: install ok installed 2024-09-13T00:06:15.301 INFO:teuthology.orchestra.run.smithi077.stdout:Priority: optional 2024-09-13T00:06:15.301 INFO:teuthology.orchestra.run.smithi077.stdout:Section: kernel 2024-09-13T00:06:15.301 INFO:teuthology.orchestra.run.smithi077.stdout:Installed-Size: 21 2024-09-13T00:06:15.301 INFO:teuthology.orchestra.run.smithi077.stdout:Maintainer: Ubuntu Kernel Team 2024-09-13T00:06:15.301 INFO:teuthology.orchestra.run.smithi077.stdout:Architecture: amd64 2024-09-13T00:06:15.301 INFO:teuthology.orchestra.run.smithi077.stdout:Source: linux-meta 2024-09-13T00:06:15.301 INFO:teuthology.orchestra.run.smithi077.stdout:Version: 5.15.0.121.121 2024-09-13T00:06:15.302 INFO:teuthology.orchestra.run.smithi077.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.4), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-121), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.4) 2024-09-13T00:06:15.302 INFO:teuthology.orchestra.run.smithi077.stdout:Depends: linux-image-5.15.0-121-generic, linux-modules-extra-5.15.0-121-generic, linux-firmware, intel-microcode, amd64-microcode 2024-09-13T00:06:15.302 INFO:teuthology.orchestra.run.smithi077.stdout:Recommends: thermald 2024-09-13T00:06:15.302 INFO:teuthology.orchestra.run.smithi077.stdout:Description: Generic Linux kernel image 2024-09-13T00:06:15.302 INFO:teuthology.orchestra.run.smithi077.stdout: This package will always depend on the latest generic kernel image 2024-09-13T00:06:15.302 INFO:teuthology.orchestra.run.smithi077.stdout: available. 2024-09-13T00:06:15.302 DEBUG:teuthology.orchestra.run.smithi077:> mktemp 2024-09-13T00:06:15.348 INFO:teuthology.orchestra.run.smithi077.stdout:/tmp/tmp.7Pfe6NLFtI 2024-09-13T00:06:15.348 DEBUG:teuthology.orchestra.run.smithi077:> sudo cp /boot/grub/grub.cfg /tmp/tmp.7Pfe6NLFtI 2024-09-13T00:06:15.355 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-13T00:06:15.355 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-09-13T00:06:15.355 DEBUG:teuthology.orchestra.run.smithi088:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-09-13T00:06:15.401 DEBUG:teuthology.orchestra.run.smithi077:> sudo chmod 0666 /tmp/tmp.7Pfe6NLFtI 2024-09-13T00:06:15.422 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-121-generic 2024-09-13T00:06:15.423 DEBUG:teuthology.orchestra.run.smithi088:> sudo update-grub 2024-09-13T00:06:15.525 DEBUG:teuthology.orchestra.remote:smithi077:/tmp/tmp.7Pfe6NLFtI is 10KB 2024-09-13T00:06:15.537 DEBUG:teuthology.orchestra.run.smithi077:> rm -fr /tmp/tmp.7Pfe6NLFtI 2024-09-13T00:06:15.542 DEBUG:teuthology.orchestra.run.smithi077:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-09-13T00:06:15.597 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:06:15.597 DEBUG:teuthology.orchestra.run.smithi077:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-09-13T00:06:15.597 DEBUG:teuthology.orchestra.run.smithi077:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-09-13T00:06:15.678 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-121-generic 2024-09-13T00:06:15.679 DEBUG:teuthology.orchestra.run.smithi077:> sudo update-grub 2024-09-13T00:06:16.724 INFO:teuthology.orchestra.run.smithi088.stderr:Sourcing file `/etc/default/grub' 2024-09-13T00:06:16.725 INFO:teuthology.orchestra.run.smithi088.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-13T00:06:16.727 INFO:teuthology.orchestra.run.smithi088.stderr:Generating grub configuration file ... 2024-09-13T00:06:16.966 INFO:teuthology.orchestra.run.smithi077.stderr:Sourcing file `/etc/default/grub' 2024-09-13T00:06:16.967 INFO:teuthology.orchestra.run.smithi077.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-09-13T00:06:16.970 INFO:teuthology.orchestra.run.smithi077.stderr:Generating grub configuration file ... 2024-09-13T00:06:16.994 INFO:teuthology.orchestra.run.smithi088.stderr:Found linux image: /boot/vmlinuz-5.15.0-121-generic 2024-09-13T00:06:17.000 INFO:teuthology.orchestra.run.smithi088.stderr:Found initrd image: /boot/initrd.img-5.15.0-121-generic 2024-09-13T00:06:17.214 INFO:teuthology.orchestra.run.smithi077.stderr:Found linux image: /boot/vmlinuz-5.15.0-121-generic 2024-09-13T00:06:17.221 INFO:teuthology.orchestra.run.smithi077.stderr:Found initrd image: /boot/initrd.img-5.15.0-121-generic 2024-09-13T00:06:17.298 INFO:teuthology.orchestra.run.smithi088.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-13T00:06:17.303 INFO:teuthology.orchestra.run.smithi088.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-13T00:06:17.346 INFO:teuthology.orchestra.run.smithi088.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-13T00:06:17.352 INFO:teuthology.orchestra.run.smithi088.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-13T00:06:17.437 INFO:teuthology.orchestra.run.smithi077.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-09-13T00:06:17.443 INFO:teuthology.orchestra.run.smithi077.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-09-13T00:06:17.483 INFO:teuthology.orchestra.run.smithi077.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-09-13T00:06:17.488 INFO:teuthology.orchestra.run.smithi077.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-09-13T00:06:17.494 INFO:teuthology.orchestra.run.smithi088.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-13T00:06:17.494 INFO:teuthology.orchestra.run.smithi088.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-09-13T00:06:17.494 INFO:teuthology.orchestra.run.smithi088.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-13T00:06:17.518 INFO:teuthology.orchestra.run.smithi088.stderr:done 2024-09-13T00:06:17.535 DEBUG:teuthology.orchestra.run.smithi088:> sudo shutdown -r now 2024-09-13T00:06:17.617 INFO:teuthology.orchestra.run.smithi077.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-09-13T00:06:17.617 INFO:teuthology.orchestra.run.smithi077.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-09-13T00:06:17.617 INFO:teuthology.orchestra.run.smithi077.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-09-13T00:06:17.629 INFO:teuthology.orchestra.run.smithi077.stderr:done 2024-09-13T00:06:17.631 DEBUG:teuthology.orchestra.run.smithi077:> sudo shutdown -r now 2024-09-13T00:06:47.565 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-13T00:06:47.566 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-09-13T00:06:47.567 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T00:06:47.634 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-13T00:06:47.634 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi077.front.sepia.ceph.com' 2024-09-13T00:06:47.635 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi077.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T00:07:06.045 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.88 2024-09-13T00:07:06.046 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.77 2024-09-13T00:07:15.055 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-09-13T00:07:15.056 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T00:07:15.058 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi077.front.sepia.ceph.com' 2024-09-13T00:07:15.058 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi077.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T00:07:18.109 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.88 2024-09-13T00:07:18.109 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.77 2024-09-13T00:07:30.121 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-09-13T00:07:30.122 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T00:07:30.124 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi077.front.sepia.ceph.com' 2024-09-13T00:07:30.124 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi077.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T00:07:33.181 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.88 2024-09-13T00:07:48.195 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-09-13T00:07:48.197 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T00:07:48.541 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.77 2024-09-13T00:08:03.553 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi077.front.sepia.ceph.com' 2024-09-13T00:08:03.554 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi077.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T00:08:04.144 DEBUG:teuthology.orchestra.run.smithi077:> true 2024-09-13T00:08:04.764 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi077.front.sepia.ceph.com' 2024-09-13T00:08:04.764 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-121-generic"... 2024-09-13T00:08:04.764 DEBUG:teuthology.orchestra.run.smithi077:> uname -r 2024-09-13T00:08:04.809 INFO:teuthology.orchestra.run.smithi077.stdout:5.15.0-121-generic 2024-09-13T00:08:04.809 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-121-generic vs 5.15.0-121-generic 2024-09-13T00:08:04.809 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T00:08:04.809 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-09-13T00:08:05.811 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-13T00:08:05.811 DEBUG:teuthology.orchestra.run.smithi077:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T00:08:05.881 INFO:teuthology.orchestra.run.smithi077.stdout:ttyS1 2024-09-13T00:08:05.919 DEBUG:teuthology.parallel:result is None 2024-09-13T00:08:20.016 DEBUG:teuthology.orchestra.run.smithi088:> true 2024-09-13T00:08:20.581 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-09-13T00:08:20.582 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-121-generic"... 2024-09-13T00:08:20.582 DEBUG:teuthology.orchestra.run.smithi088:> uname -r 2024-09-13T00:08:20.628 INFO:teuthology.orchestra.run.smithi088.stdout:5.15.0-121-generic 2024-09-13T00:08:20.628 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-121-generic vs 5.15.0-121-generic 2024-09-13T00:08:20.628 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-13T00:08:20.629 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-09-13T00:08:21.630 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-13T00:08:21.630 DEBUG:teuthology.orchestra.run.smithi088:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-13T00:08:21.695 INFO:teuthology.orchestra.run.smithi088.stdout:ttyS1 2024-09-13T00:08:21.717 DEBUG:teuthology.parallel:result is None 2024-09-13T00:08:21.718 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-13T00:08:21.726 INFO:teuthology.task.internal:Creating test directory... 2024-09-13T00:08:21.726 DEBUG:teuthology.orchestra.run.smithi077:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T00:08:21.729 DEBUG:teuthology.orchestra.run.smithi088:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-13T00:08:21.734 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-13T00:08:21.740 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-13T00:08:21.747 INFO:teuthology.task.internal:Creating archive directory... 2024-09-13T00:08:21.747 DEBUG:teuthology.orchestra.run.smithi077:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T00:08:21.778 DEBUG:teuthology.orchestra.run.smithi088:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-13T00:08:21.937 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-13T00:08:21.943 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-13T00:08:21.943 DEBUG:teuthology.orchestra.run.smithi077:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T00:08:21.945 DEBUG:teuthology.orchestra.run.smithi088:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-13T00:08:21.962 INFO:teuthology.orchestra.run.smithi077.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T00:08:21.967 INFO:teuthology.orchestra.run.smithi088.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T00:08:21.969 INFO:teuthology.orchestra.run.smithi077.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T00:08:21.975 INFO:teuthology.orchestra.run.smithi088.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-13T00:08:21.977 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-13T00:08:22.036 INFO:teuthology.task.internal:Configuring sudo... 2024-09-13T00:08:22.036 DEBUG:teuthology.orchestra.run.smithi077:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T00:08:22.039 DEBUG:teuthology.orchestra.run.smithi088:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-13T00:08:22.052 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-13T00:08:22.063 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-13T00:08:22.063 DEBUG:teuthology.orchestra.run.smithi077:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T00:08:22.098 DEBUG:teuthology.orchestra.run.smithi088:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-13T00:08:22.103 DEBUG:teuthology.orchestra.run.smithi077:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T00:08:22.149 DEBUG:teuthology.orchestra.run.smithi077:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T00:08:22.197 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:08:22.197 DEBUG:teuthology.orchestra.run.smithi077:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T00:08:22.270 DEBUG:teuthology.orchestra.run.smithi088:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-13T00:08:22.276 DEBUG:teuthology.orchestra.run.smithi088:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-13T00:08:22.324 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-13T00:08:22.324 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-13T00:08:22.388 DEBUG:teuthology.orchestra.run.smithi077:> sudo service rsyslog restart 2024-09-13T00:08:22.390 DEBUG:teuthology.orchestra.run.smithi088:> sudo service rsyslog restart 2024-09-13T00:08:22.461 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-13T00:08:22.469 INFO:teuthology.task.internal:Starting timer... 2024-09-13T00:08:22.469 INFO:teuthology.run_tasks:Running task pcp... 2024-09-13T00:08:22.479 INFO:teuthology.run_tasks:Running task selinux... 2024-09-13T00:08:22.489 DEBUG:teuthology.task.selinux:Excluding smithi077: OS 'ubuntu' does not support SELinux 2024-09-13T00:08:22.490 DEBUG:teuthology.task.selinux:Excluding smithi088: OS 'ubuntu' does not support SELinux 2024-09-13T00:08:22.490 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-13T00:08:22.490 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2024-09-13T00:08:22.490 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-13T00:08:22.490 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-13T00:08:22.498 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-09-13T00:08:22.502 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-09-13T00:08:22.615 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-13T00:08:22.626 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-13T00:08:22.628 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi077.front.sepia.ceph.com,smithi088.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-13T00:15:49.917 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi077.front.sepia.ceph.com'), Remote(name='ubuntu@smithi088.front.sepia.ceph.com')] 2024-09-13T00:15:49.919 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi077.front.sepia.ceph.com' 2024-09-13T00:15:49.921 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi077.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T00:15:50.005 DEBUG:teuthology.orchestra.run.smithi077:> true 2024-09-13T00:15:50.081 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi077.front.sepia.ceph.com' 2024-09-13T00:15:50.081 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-09-13T00:15:50.082 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi088.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-13T00:15:50.160 DEBUG:teuthology.orchestra.run.smithi088:> true 2024-09-13T00:15:50.237 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi088.front.sepia.ceph.com' 2024-09-13T00:15:50.237 INFO:teuthology.run_tasks:Running task clock... 2024-09-13T00:15:50.248 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-13T00:15:50.249 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T00:15:50.250 DEBUG:teuthology.orchestra.run.smithi077:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T00:15:50.253 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-13T00:15:50.253 DEBUG:teuthology.orchestra.run.smithi088:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T00:15:50.277 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-09-13T00:15:50.277 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: Command line: ntpd -gq 2024-09-13T00:15:50.277 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: ---------------------------------------------------- 2024-09-13T00:15:50.277 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: ntp-4 is maintained by Network Time Foundation, 2024-09-13T00:15:50.277 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-09-13T00:15:50.277 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: corporation. Support and training for ntp-4 are 2024-09-13T00:15:50.277 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: available at https://www.nwtime.org/support 2024-09-13T00:15:50.277 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: ---------------------------------------------------- 2024-09-13T00:15:50.278 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: proto: precision = 0.046 usec (-24) 2024-09-13T00:15:50.278 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: basedate set to 2022-02-04 2024-09-13T00:15:50.278 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: gps base set to 2022-02-06 (week 2196) 2024-09-13T00:15:50.278 INFO:teuthology.orchestra.run.smithi077.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-13T00:15:50.279 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-13T00:15:50.279 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: restrict ::: KOD does nothing without LIMITED. 2024-09-13T00:15:50.279 INFO:teuthology.orchestra.run.smithi077.stderr:restrict ::: KOD does nothing without LIMITED. 2024-09-13T00:15:50.280 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: Listen and drop on 0 v6wildcard [::]:123 2024-09-13T00:15:50.280 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-09-13T00:15:50.280 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: Listen normally on 2 lo 127.0.0.1:123 2024-09-13T00:15:50.280 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: Listen normally on 3 enp3s0f1 172.21.15.77:123 2024-09-13T00:15:50.280 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: Listen normally on 4 lo [::1]:123 2024-09-13T00:15:50.280 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe8f:d3d3%5]:123 2024-09-13T00:15:50.281 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:50 ntpd[18468]: Listening on routing socket on fd #22 for interface updates 2024-09-13T00:15:50.305 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-09-13T00:15:50.305 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: Command line: ntpd -gq 2024-09-13T00:15:50.305 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: ---------------------------------------------------- 2024-09-13T00:15:50.305 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: ntp-4 is maintained by Network Time Foundation, 2024-09-13T00:15:50.305 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-09-13T00:15:50.305 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: corporation. Support and training for ntp-4 are 2024-09-13T00:15:50.305 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: available at https://www.nwtime.org/support 2024-09-13T00:15:50.305 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: ---------------------------------------------------- 2024-09-13T00:15:50.306 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: proto: precision = 0.045 usec (-24) 2024-09-13T00:15:50.306 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: basedate set to 2022-02-04 2024-09-13T00:15:50.306 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: gps base set to 2022-02-06 (week 2196) 2024-09-13T00:15:50.306 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-13T00:15:50.307 INFO:teuthology.orchestra.run.smithi088.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-09-13T00:15:50.307 INFO:teuthology.orchestra.run.smithi088.stderr:restrict ::: KOD does nothing without LIMITED. 2024-09-13T00:15:50.308 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: restrict ::: KOD does nothing without LIMITED. 2024-09-13T00:15:50.308 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: Listen and drop on 0 v6wildcard [::]:123 2024-09-13T00:15:50.308 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-09-13T00:15:50.308 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: Listen normally on 2 lo 127.0.0.1:123 2024-09-13T00:15:50.308 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: Listen normally on 3 enp3s0f1 172.21.15.88:123 2024-09-13T00:15:50.308 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: Listen normally on 4 lo [::1]:123 2024-09-13T00:15:50.308 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe8f:d42d%5]:123 2024-09-13T00:15:50.308 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:50 ntpd[18480]: Listening on routing socket on fd #22 for interface updates 2024-09-13T00:15:51.277 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:51 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:51.277 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:51 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:51.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:51 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:51.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:51 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:51.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:51 ntpd[18468]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:51.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:51 ntpd[18468]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:51.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:51.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:51.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:51.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:51.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:51.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:51.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:51.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:51.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:51.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:51.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:51.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:51.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:51.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:51.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:51.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:51.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:51.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:51 ntpd[18480]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:52.277 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:52 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:52.277 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:52 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:53.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:53 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:53.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:53 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:53.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:53 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:53.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:53 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:53.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:53 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:53.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:53 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:53.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:53 ntpd[18468]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:53.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:53 ntpd[18468]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:53.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:53 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:53.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:53 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:53.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:53 ntpd[18468]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:53.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:53 ntpd[18468]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:53.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:53.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:53.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:53.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:53.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:53.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:53.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:53.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:53.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:53.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:53.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:53.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:53.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:53.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:53.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:53.308 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:53.308 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:53.308 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:53 ntpd[18480]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:54.305 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:54 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:54.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:54 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:55.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:55.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:55.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:55.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:55.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:55.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:55.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:55.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:55.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:55.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:55.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:55.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:55.280 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:55.280 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:55.280 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:55.280 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:55.280 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:55.280 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:55 ntpd[18468]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:55.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:55.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:55.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:55.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:55.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:55.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:55.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:55.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:55.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:55.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:55.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:55.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:55.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:55.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:55.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:55.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:55.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:55.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:55 ntpd[18480]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:57.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:57 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:57.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:57 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:57.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:57 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:57.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:57 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:57.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:57 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:57.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:57 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:57.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:57 ntpd[18468]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:57.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:57 ntpd[18468]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:57.278 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:57 ntpd[18468]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:57.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:57 ntpd[18468]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:57.279 INFO:teuthology.orchestra.run.smithi077.stdout:13 Sep 00:15:57 ntpd[18468]: ntpd: time slew -0.000924 s 2024-09-13T00:15:57.279 INFO:teuthology.orchestra.run.smithi077.stdout:ntpd: time slew -0.000924s 2024-09-13T00:15:57.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:57 ntpd[18468]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-09-13T00:15:57.279 INFO:teuthology.orchestra.run.smithi077.stderr:13 Sep 00:15:57 ntpd[18468]: can't open /var/log/ntpstats/loopstats.20240913: Permission denied 2024-09-13T00:15:57.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:57 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:57.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:57 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:57.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:57 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:57.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:57 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:57.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:57 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:57.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:57 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:57.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:57 ntpd[18480]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-09-13T00:15:57.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:57 ntpd[18480]: can't open /var/log/ntpstats/rawstats.20240913: Permission denied 2024-09-13T00:15:57.306 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:57 ntpd[18480]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-09-13T00:15:57.307 INFO:teuthology.orchestra.run.smithi088.stdout:13 Sep 00:15:57 ntpd[18480]: ntpd: time slew -0.001035 s 2024-09-13T00:15:57.307 INFO:teuthology.orchestra.run.smithi088.stdout:ntpd: time slew -0.001035s 2024-09-13T00:15:57.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:57 ntpd[18480]: can't open /var/log/ntpstats/peerstats.20240913: Permission denied 2024-09-13T00:15:57.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:57 ntpd[18480]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-09-13T00:15:57.307 INFO:teuthology.orchestra.run.smithi088.stderr:13 Sep 00:15:57 ntpd[18480]: can't open /var/log/ntpstats/loopstats.20240913: Permission denied 2024-09-13T00:15:57.336 INFO:teuthology.orchestra.run.smithi077.stdout: remote refid st t when poll reach delay offset jitter 2024-09-13T00:15:57.336 INFO:teuthology.orchestra.run.smithi077.stdout:============================================================================== 2024-09-13T00:15:57.336 INFO:teuthology.orchestra.run.smithi077.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-13T00:15:57.336 INFO:teuthology.orchestra.run.smithi077.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-13T00:15:57.337 INFO:teuthology.orchestra.run.smithi077.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-13T00:15:57.337 INFO:teuthology.orchestra.run.smithi077.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-13T00:15:57.367 INFO:teuthology.orchestra.run.smithi088.stdout: remote refid st t when poll reach delay offset jitter 2024-09-13T00:15:57.368 INFO:teuthology.orchestra.run.smithi088.stdout:============================================================================== 2024-09-13T00:15:57.368 INFO:teuthology.orchestra.run.smithi088.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-13T00:15:57.368 INFO:teuthology.orchestra.run.smithi088.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-13T00:15:57.368 INFO:teuthology.orchestra.run.smithi088.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-13T00:15:57.368 INFO:teuthology.orchestra.run.smithi088.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-09-13T00:15:57.369 INFO:teuthology.run_tasks:Running task install... 2024-09-13T00:15:57.378 DEBUG:teuthology.task.install:project ceph 2024-09-13T00:15:57.378 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '7a853b1706d076d58e0da7a5d3b9b9eb36276880'}} 2024-09-13T00:15:57.379 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '7a853b1706d076d58e0da7a5d3b9b9eb36276880'} 2024-09-13T00:15:57.379 INFO:teuthology.task.install:Using flavor: default 2024-09-13T00:15:57.384 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-13T00:15:57.384 INFO:teuthology.task.install:extra packages: [] 2024-09-13T00:15:57.385 DEBUG:teuthology.orchestra.run.smithi077:> sudo apt-key list | grep Ceph 2024-09-13T00:15:57.385 DEBUG:teuthology.orchestra.run.smithi088:> sudo apt-key list | grep Ceph 2024-09-13T00:15:57.439 INFO:teuthology.orchestra.run.smithi077.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-09-13T00:15:57.465 INFO:teuthology.orchestra.run.smithi077.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-09-13T00:15:57.465 INFO:teuthology.orchestra.run.smithi077.stdout:uid [ unknown] Ceph.com (release key) 2024-09-13T00:15:57.466 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2024-09-13T00:15:57.466 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T00:15:57.466 INFO:teuthology.packaging:ref: None 2024-09-13T00:15:57.466 INFO:teuthology.packaging:tag: None 2024-09-13T00:15:57.466 INFO:teuthology.packaging:branch: quincy 2024-09-13T00:15:57.466 INFO:teuthology.packaging:sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-13T00:15:57.467 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-09-13T00:15:57.476 INFO:teuthology.orchestra.run.smithi088.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-09-13T00:15:57.505 INFO:teuthology.orchestra.run.smithi088.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-09-13T00:15:57.505 INFO:teuthology.orchestra.run.smithi088.stdout:uid [ unknown] Ceph.com (release key) 2024-09-13T00:15:57.505 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2024-09-13T00:15:57.505 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T00:15:57.505 INFO:teuthology.packaging:ref: None 2024-09-13T00:15:57.506 INFO:teuthology.packaging:tag: None 2024-09-13T00:15:57.506 INFO:teuthology.packaging:branch: quincy 2024-09-13T00:15:57.506 INFO:teuthology.packaging:sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-13T00:15:57.506 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-09-13T00:15:57.611 INFO:teuthology.task.install.deb:Pulling from https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default/ 2024-09-13T00:15:57.611 INFO:teuthology.task.install.deb:Package version is 17.2.7-1jammy 2024-09-13T00:15:57.642 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:15:57.642 DEBUG:teuthology.orchestra.run.smithi077:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-09-13T00:15:57.644 INFO:teuthology.task.install.deb:Pulling from https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default/ 2024-09-13T00:15:57.644 INFO:teuthology.task.install.deb:Package version is 17.2.7-1jammy 2024-09-13T00:15:57.671 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-13T00:15:57.671 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-09-13T00:15:57.683 DEBUG:teuthology.orchestra.run.smithi088:> sudo apt-get update 2024-09-13T00:15:57.691 DEBUG:teuthology.orchestra.run.smithi077:> sudo apt-get update 2024-09-13T00:15:57.823 INFO:teuthology.orchestra.run.smithi088.stdout:Ign:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy InRelease 2024-09-13T00:15:57.824 INFO:teuthology.orchestra.run.smithi088.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release [7,641 B] 2024-09-13T00:15:57.826 INFO:teuthology.orchestra.run.smithi088.stdout:Ign:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release.gpg 2024-09-13T00:15:57.827 INFO:teuthology.orchestra.run.smithi088.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,331 B] 2024-09-13T00:15:57.832 INFO:teuthology.orchestra.run.smithi077.stdout:Ign:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy InRelease 2024-09-13T00:15:57.833 INFO:teuthology.orchestra.run.smithi077.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release [7,641 B] 2024-09-13T00:15:57.835 INFO:teuthology.orchestra.run.smithi077.stdout:Ign:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy Release.gpg 2024-09-13T00:15:57.835 INFO:teuthology.orchestra.run.smithi088.stdout:Get:5 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.3 kB] 2024-09-13T00:15:57.836 INFO:teuthology.orchestra.run.smithi077.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,331 B] 2024-09-13T00:15:57.844 INFO:teuthology.orchestra.run.smithi077.stdout:Get:5 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.3 kB] 2024-09-13T00:15:57.848 INFO:teuthology.orchestra.run.smithi088.stdout:Hit:6 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-13T00:15:57.848 INFO:teuthology.orchestra.run.smithi088.stdout:Hit:7 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-13T00:15:57.859 INFO:teuthology.orchestra.run.smithi088.stdout:Hit:8 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-13T00:15:57.861 INFO:teuthology.orchestra.run.smithi077.stdout:Hit:6 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-09-13T00:15:57.871 INFO:teuthology.orchestra.run.smithi077.stdout:Hit:7 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-09-13T00:15:57.876 INFO:teuthology.orchestra.run.smithi088.stdout:Hit:9 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-13T00:15:57.888 INFO:teuthology.orchestra.run.smithi077.stdout:Hit:8 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-09-13T00:15:57.983 INFO:teuthology.orchestra.run.smithi077.stdout:Hit:9 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-09-13T00:15:58.188 INFO:teuthology.orchestra.run.smithi088.stdout:Fetched 28.3 kB in 0s (86.1 kB/s) 2024-09-13T00:15:58.221 INFO:teuthology.orchestra.run.smithi077.stdout:Fetched 28.3 kB in 0s (81.7 kB/s) 2024-09-13T00:15:59.211 INFO:teuthology.orchestra.run.smithi088.stdout:Reading package lists... 2024-09-13T00:15:59.226 DEBUG:teuthology.orchestra.run.smithi088:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=17.2.7-1jammy cephadm=17.2.7-1jammy ceph-mds=17.2.7-1jammy ceph-mgr=17.2.7-1jammy ceph-common=17.2.7-1jammy ceph-fuse=17.2.7-1jammy ceph-test=17.2.7-1jammy radosgw=17.2.7-1jammy python3-rados=17.2.7-1jammy python3-rgw=17.2.7-1jammy python3-cephfs=17.2.7-1jammy python3-rbd=17.2.7-1jammy libcephfs2=17.2.7-1jammy libcephfs-dev=17.2.7-1jammy librados2=17.2.7-1jammy librbd1=17.2.7-1jammy rbd-fuse=17.2.7-1jammy 2024-09-13T00:15:59.239 INFO:teuthology.orchestra.run.smithi077.stdout:Reading package lists... 2024-09-13T00:15:59.256 DEBUG:teuthology.orchestra.run.smithi077:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=17.2.7-1jammy cephadm=17.2.7-1jammy ceph-mds=17.2.7-1jammy ceph-mgr=17.2.7-1jammy ceph-common=17.2.7-1jammy ceph-fuse=17.2.7-1jammy ceph-test=17.2.7-1jammy radosgw=17.2.7-1jammy python3-rados=17.2.7-1jammy python3-rgw=17.2.7-1jammy python3-cephfs=17.2.7-1jammy python3-rbd=17.2.7-1jammy libcephfs2=17.2.7-1jammy libcephfs-dev=17.2.7-1jammy librados2=17.2.7-1jammy librbd1=17.2.7-1jammy rbd-fuse=17.2.7-1jammy 2024-09-13T00:15:59.288 INFO:teuthology.orchestra.run.smithi088.stdout:Reading package lists... 2024-09-13T00:15:59.320 INFO:teuthology.orchestra.run.smithi077.stdout:Reading package lists... 2024-09-13T00:15:59.498 INFO:teuthology.orchestra.run.smithi088.stdout:Building dependency tree... 2024-09-13T00:15:59.498 INFO:teuthology.orchestra.run.smithi088.stdout:Reading state information... 2024-09-13T00:15:59.529 INFO:teuthology.orchestra.run.smithi077.stdout:Building dependency tree... 2024-09-13T00:15:59.530 INFO:teuthology.orchestra.run.smithi077.stdout:Reading state information... 2024-09-13T00:15:59.691 INFO:teuthology.orchestra.run.smithi088.stdout:The following packages were automatically installed and are no longer required: 2024-09-13T00:15:59.691 INFO:teuthology.orchestra.run.smithi088.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-13T00:15:59.691 INFO:teuthology.orchestra.run.smithi088.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-13T00:15:59.691 INFO:teuthology.orchestra.run.smithi088.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-13T00:15:59.692 INFO:teuthology.orchestra.run.smithi088.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-13T00:15:59.692 INFO:teuthology.orchestra.run.smithi088.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-13T00:15:59.692 INFO:teuthology.orchestra.run.smithi088.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-13T00:15:59.692 INFO:teuthology.orchestra.run.smithi088.stdout:The following additional packages will be installed: 2024-09-13T00:15:59.692 INFO:teuthology.orchestra.run.smithi088.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-09-13T00:15:59.692 INFO:teuthology.orchestra.run.smithi088.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume jq 2024-09-13T00:15:59.693 INFO:teuthology.orchestra.run.smithi088.stdout: libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 liboath0 libonig5 2024-09-13T00:15:59.694 INFO:teuthology.orchestra.run.smithi088.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-09-13T00:15:59.694 INFO:teuthology.orchestra.run.smithi088.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli 2024-09-13T00:15:59.694 INFO:teuthology.orchestra.run.smithi088.stdout: python-babel-localedata python-pastedeploy-tpl python3-babel python3-bcrypt 2024-09-13T00:15:59.694 INFO:teuthology.orchestra.run.smithi088.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-certifi 2024-09-13T00:15:59.694 INFO:teuthology.orchestra.run.smithi088.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-idna 2024-09-13T00:15:59.694 INFO:teuthology.orchestra.run.smithi088.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2024-09-13T00:15:59.694 INFO:teuthology.orchestra.run.smithi088.stdout: python3-jaraco.functools python3-jaraco.text python3-jinja2 python3-joblib 2024-09-13T00:15:59.694 INFO:teuthology.orchestra.run.smithi088.stdout: python3-kubernetes python3-logutils python3-mako python3-markupsafe 2024-09-13T00:15:59.694 INFO:teuthology.orchestra.run.smithi088.stdout: python3-natsort python3-openssl python3-paste python3-pastedeploy 2024-09-13T00:15:59.694 INFO:teuthology.orchestra.run.smithi088.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2024-09-13T00:15:59.694 INFO:teuthology.orchestra.run.smithi088.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2024-09-13T00:15:59.694 INFO:teuthology.orchestra.run.smithi088.stdout: python3-pyasn1-modules python3-pygments python3-pyinotify python3-pytest 2024-09-13T00:15:59.694 INFO:teuthology.orchestra.run.smithi088.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-routes 2024-09-13T00:15:59.695 INFO:teuthology.orchestra.run.smithi088.stdout: python3-rsa python3-simplegeneric python3-simplejson python3-singledispatch 2024-09-13T00:15:59.695 INFO:teuthology.orchestra.run.smithi088.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-09-13T00:15:59.695 INFO:teuthology.orchestra.run.smithi088.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-09-13T00:15:59.695 INFO:teuthology.orchestra.run.smithi088.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-09-13T00:15:59.695 INFO:teuthology.orchestra.run.smithi088.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat xmlstarlet 2024-09-13T00:15:59.695 INFO:teuthology.orchestra.run.smithi088.stdout:Suggested packages: 2024-09-13T00:15:59.695 INFO:teuthology.orchestra.run.smithi088.stdout: python3-influxdb python-jinja2-doc python3-beaker python-mako-doc 2024-09-13T00:15:59.695 INFO:teuthology.orchestra.run.smithi088.stdout: python-natsort-doc python-openssl-doc python3-openssl-dbg httpd-wsgi 2024-09-13T00:15:59.695 INFO:teuthology.orchestra.run.smithi088.stdout: libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc 2024-09-13T00:15:59.696 INFO:teuthology.orchestra.run.smithi088.stdout: python-psutil-doc subversion python-pygments-doc python-pyinotify-doc 2024-09-13T00:15:59.696 INFO:teuthology.orchestra.run.smithi088.stdout: python3-socks python-requests-doc python3-dap python-sklearn-doc ipython3 2024-09-13T00:15:59.696 INFO:teuthology.orchestra.run.smithi088.stdout: python-waitress-doc python-webob-doc python-webtest-doc python-werkzeug-doc 2024-09-13T00:15:59.696 INFO:teuthology.orchestra.run.smithi088.stdout: python3-watchdog 2024-09-13T00:15:59.696 INFO:teuthology.orchestra.run.smithi088.stdout:Recommended packages: 2024-09-13T00:15:59.696 INFO:teuthology.orchestra.run.smithi088.stdout: btrfs-tools 2024-09-13T00:15:59.720 INFO:teuthology.orchestra.run.smithi077.stdout:The following packages were automatically installed and are no longer required: 2024-09-13T00:15:59.720 INFO:teuthology.orchestra.run.smithi077.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-09-13T00:15:59.720 INFO:teuthology.orchestra.run.smithi077.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-09-13T00:15:59.720 INFO:teuthology.orchestra.run.smithi077.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-09-13T00:15:59.721 INFO:teuthology.orchestra.run.smithi077.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-09-13T00:15:59.721 INFO:teuthology.orchestra.run.smithi077.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-09-13T00:15:59.721 INFO:teuthology.orchestra.run.smithi077.stdout:Use 'sudo apt autoremove' to remove them. 2024-09-13T00:15:59.722 INFO:teuthology.orchestra.run.smithi077.stdout:The following additional packages will be installed: 2024-09-13T00:15:59.722 INFO:teuthology.orchestra.run.smithi077.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-09-13T00:15:59.722 INFO:teuthology.orchestra.run.smithi077.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume jq 2024-09-13T00:15:59.722 INFO:teuthology.orchestra.run.smithi077.stdout: libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 liboath0 libonig5 2024-09-13T00:15:59.722 INFO:teuthology.orchestra.run.smithi077.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-09-13T00:15:59.723 INFO:teuthology.orchestra.run.smithi077.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli 2024-09-13T00:15:59.723 INFO:teuthology.orchestra.run.smithi077.stdout: python-babel-localedata python-pastedeploy-tpl python3-babel python3-bcrypt 2024-09-13T00:15:59.723 INFO:teuthology.orchestra.run.smithi077.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-certifi 2024-09-13T00:15:59.723 INFO:teuthology.orchestra.run.smithi077.stdout: python3-cheroot python3-cherrypy3 python3-google-auth python3-idna 2024-09-13T00:15:59.723 INFO:teuthology.orchestra.run.smithi077.stdout: python3-iniconfig python3-jaraco.classes python3-jaraco.collections 2024-09-13T00:15:59.723 INFO:teuthology.orchestra.run.smithi077.stdout: python3-jaraco.functools python3-jaraco.text python3-jinja2 python3-joblib 2024-09-13T00:15:59.723 INFO:teuthology.orchestra.run.smithi077.stdout: python3-kubernetes python3-logutils python3-mako python3-markupsafe 2024-09-13T00:15:59.723 INFO:teuthology.orchestra.run.smithi077.stdout: python3-natsort python3-openssl python3-paste python3-pastedeploy 2024-09-13T00:15:59.723 INFO:teuthology.orchestra.run.smithi077.stdout: python3-pastescript python3-pecan python3-pluggy python3-portend 2024-09-13T00:15:59.723 INFO:teuthology.orchestra.run.smithi077.stdout: python3-prettytable python3-psutil python3-py python3-pyasn1 2024-09-13T00:15:59.724 INFO:teuthology.orchestra.run.smithi077.stdout: python3-pyasn1-modules python3-pygments python3-pyinotify python3-pytest 2024-09-13T00:15:59.724 INFO:teuthology.orchestra.run.smithi077.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-routes 2024-09-13T00:15:59.724 INFO:teuthology.orchestra.run.smithi077.stdout: python3-rsa python3-simplegeneric python3-simplejson python3-singledispatch 2024-09-13T00:15:59.724 INFO:teuthology.orchestra.run.smithi077.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-09-13T00:15:59.724 INFO:teuthology.orchestra.run.smithi077.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-09-13T00:15:59.724 INFO:teuthology.orchestra.run.smithi077.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-09-13T00:15:59.724 INFO:teuthology.orchestra.run.smithi077.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat xmlstarlet 2024-09-13T00:15:59.724 INFO:teuthology.orchestra.run.smithi077.stdout:Suggested packages: 2024-09-13T00:15:59.725 INFO:teuthology.orchestra.run.smithi077.stdout: python3-influxdb python-jinja2-doc python3-beaker python-mako-doc 2024-09-13T00:15:59.725 INFO:teuthology.orchestra.run.smithi077.stdout: python-natsort-doc python-openssl-doc python3-openssl-dbg httpd-wsgi 2024-09-13T00:15:59.725 INFO:teuthology.orchestra.run.smithi077.stdout: libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc 2024-09-13T00:15:59.725 INFO:teuthology.orchestra.run.smithi077.stdout: python-psutil-doc subversion python-pygments-doc python-pyinotify-doc 2024-09-13T00:15:59.725 INFO:teuthology.orchestra.run.smithi077.stdout: python3-socks python-requests-doc python3-dap python-sklearn-doc ipython3 2024-09-13T00:15:59.725 INFO:teuthology.orchestra.run.smithi077.stdout: python-waitress-doc python-webob-doc python-webtest-doc python-werkzeug-doc 2024-09-13T00:15:59.725 INFO:teuthology.orchestra.run.smithi077.stdout: python3-watchdog 2024-09-13T00:15:59.725 INFO:teuthology.orchestra.run.smithi077.stdout:Recommended packages: 2024-09-13T00:15:59.725 INFO:teuthology.orchestra.run.smithi077.stdout: btrfs-tools 2024-09-13T00:15:59.794 INFO:teuthology.orchestra.run.smithi088.stdout:The following NEW packages will be installed: 2024-09-13T00:15:59.794 INFO:teuthology.orchestra.run.smithi088.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-09-13T00:15:59.794 INFO:teuthology.orchestra.run.smithi088.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-09-13T00:15:59.795 INFO:teuthology.orchestra.run.smithi088.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-09-13T00:15:59.796 INFO:teuthology.orchestra.run.smithi088.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 2024-09-13T00:15:59.796 INFO:teuthology.orchestra.run.smithi088.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-09-13T00:15:59.796 INFO:teuthology.orchestra.run.smithi088.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2024-09-13T00:15:59.796 INFO:teuthology.orchestra.run.smithi088.stdout: libthrift-0.16.0 nvme-cli python-babel-localedata python-pastedeploy-tpl 2024-09-13T00:15:59.796 INFO:teuthology.orchestra.run.smithi088.stdout: python3-babel python3-bcrypt python3-cachetools python3-ceph-argparse 2024-09-13T00:15:59.796 INFO:teuthology.orchestra.run.smithi088.stdout: python3-ceph-common python3-cephfs python3-certifi python3-cheroot 2024-09-13T00:15:59.796 INFO:teuthology.orchestra.run.smithi088.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-09-13T00:15:59.796 INFO:teuthology.orchestra.run.smithi088.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-09-13T00:15:59.796 INFO:teuthology.orchestra.run.smithi088.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-09-13T00:15:59.797 INFO:teuthology.orchestra.run.smithi088.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-09-13T00:15:59.797 INFO:teuthology.orchestra.run.smithi088.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-09-13T00:15:59.797 INFO:teuthology.orchestra.run.smithi088.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-09-13T00:15:59.797 INFO:teuthology.orchestra.run.smithi088.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-09-13T00:15:59.797 INFO:teuthology.orchestra.run.smithi088.stdout: python3-pygments python3-pyinotify python3-pytest python3-rados python3-rbd 2024-09-13T00:15:59.797 INFO:teuthology.orchestra.run.smithi088.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-rgw 2024-09-13T00:15:59.797 INFO:teuthology.orchestra.run.smithi088.stdout: python3-routes python3-rsa python3-simplegeneric python3-simplejson 2024-09-13T00:15:59.797 INFO:teuthology.orchestra.run.smithi088.stdout: python3-singledispatch python3-sklearn python3-sklearn-lib python3-tempita 2024-09-13T00:15:59.797 INFO:teuthology.orchestra.run.smithi088.stdout: python3-tempora python3-threadpoolctl python3-toml python3-urllib3 2024-09-13T00:15:59.798 INFO:teuthology.orchestra.run.smithi088.stdout: python3-waitress python3-wcwidth python3-webob python3-websocket 2024-09-13T00:15:59.798 INFO:teuthology.orchestra.run.smithi088.stdout: python3-webtest python3-werkzeug python3-zc.lockfile qttranslations5-l10n 2024-09-13T00:15:59.798 INFO:teuthology.orchestra.run.smithi088.stdout: radosgw rbd-fuse socat xmlstarlet 2024-09-13T00:15:59.824 INFO:teuthology.orchestra.run.smithi077.stdout:The following NEW packages will be installed: 2024-09-13T00:15:59.824 INFO:teuthology.orchestra.run.smithi077.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-09-13T00:15:59.824 INFO:teuthology.orchestra.run.smithi077.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-09-13T00:15:59.824 INFO:teuthology.orchestra.run.smithi077.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-09-13T00:15:59.824 INFO:teuthology.orchestra.run.smithi077.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 2024-09-13T00:15:59.825 INFO:teuthology.orchestra.run.smithi077.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-09-13T00:15:59.825 INFO:teuthology.orchestra.run.smithi077.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2024-09-13T00:15:59.825 INFO:teuthology.orchestra.run.smithi077.stdout: libthrift-0.16.0 nvme-cli python-babel-localedata python-pastedeploy-tpl 2024-09-13T00:15:59.825 INFO:teuthology.orchestra.run.smithi077.stdout: python3-babel python3-bcrypt python3-cachetools python3-ceph-argparse 2024-09-13T00:15:59.826 INFO:teuthology.orchestra.run.smithi077.stdout: python3-ceph-common python3-cephfs python3-certifi python3-cheroot 2024-09-13T00:15:59.826 INFO:teuthology.orchestra.run.smithi077.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-09-13T00:15:59.826 INFO:teuthology.orchestra.run.smithi077.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-09-13T00:15:59.826 INFO:teuthology.orchestra.run.smithi077.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-09-13T00:15:59.826 INFO:teuthology.orchestra.run.smithi077.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-09-13T00:15:59.826 INFO:teuthology.orchestra.run.smithi077.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-09-13T00:15:59.826 INFO:teuthology.orchestra.run.smithi077.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-09-13T00:15:59.826 INFO:teuthology.orchestra.run.smithi077.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-09-13T00:15:59.826 INFO:teuthology.orchestra.run.smithi077.stdout: python3-pygments python3-pyinotify python3-pytest python3-rados python3-rbd 2024-09-13T00:15:59.826 INFO:teuthology.orchestra.run.smithi077.stdout: python3-repoze.lru python3-requests python3-requests-oauthlib python3-rgw 2024-09-13T00:15:59.826 INFO:teuthology.orchestra.run.smithi077.stdout: python3-routes python3-rsa python3-simplegeneric python3-simplejson 2024-09-13T00:15:59.826 INFO:teuthology.orchestra.run.smithi077.stdout: python3-singledispatch python3-sklearn python3-sklearn-lib python3-tempita 2024-09-13T00:15:59.827 INFO:teuthology.orchestra.run.smithi077.stdout: python3-tempora python3-threadpoolctl python3-toml python3-urllib3 2024-09-13T00:15:59.827 INFO:teuthology.orchestra.run.smithi077.stdout: python3-waitress python3-wcwidth python3-webob python3-websocket 2024-09-13T00:15:59.827 INFO:teuthology.orchestra.run.smithi077.stdout: python3-webtest python3-werkzeug python3-zc.lockfile qttranslations5-l10n 2024-09-13T00:15:59.827 INFO:teuthology.orchestra.run.smithi077.stdout: radosgw rbd-fuse socat xmlstarlet 2024-09-13T00:15:59.840 INFO:teuthology.orchestra.run.smithi088.stdout:0 upgraded, 107 newly installed, 0 to remove and 331 not upgraded. 2024-09-13T00:15:59.840 INFO:teuthology.orchestra.run.smithi088.stdout:Need to get 187 MB of archives. 2024-09-13T00:15:59.840 INFO:teuthology.orchestra.run.smithi088.stdout:After this operation, 815 MB of additional disk space will be used. 2024-09-13T00:15:59.840 INFO:teuthology.orchestra.run.smithi088.stdout:Get:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 17.2.7-1jammy [3,198 kB] 2024-09-13T00:15:59.859 INFO:teuthology.orchestra.run.smithi088.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfmt8 amd64 8.1.1+ds1-2 [60.2 kB] 2024-09-13T00:15:59.865 INFO:teuthology.orchestra.run.smithi077.stdout:0 upgraded, 107 newly installed, 0 to remove and 331 not upgraded. 2024-09-13T00:15:59.865 INFO:teuthology.orchestra.run.smithi077.stdout:Need to get 187 MB of archives. 2024-09-13T00:15:59.865 INFO:teuthology.orchestra.run.smithi077.stdout:After this operation, 815 MB of additional disk space will be used. 2024-09-13T00:15:59.865 INFO:teuthology.orchestra.run.smithi077.stdout:Get:1 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 17.2.7-1jammy [3,198 kB] 2024-09-13T00:15:59.870 INFO:teuthology.orchestra.run.smithi088.stdout:Get:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 17.2.7-1jammy [3,067 kB] 2024-09-13T00:15:59.893 INFO:teuthology.orchestra.run.smithi077.stdout:Get:2 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 17.2.7-1jammy [3,067 kB] 2024-09-13T00:15:59.894 INFO:teuthology.orchestra.run.smithi088.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 17.2.7-1jammy [632 kB] 2024-09-13T00:15:59.899 INFO:teuthology.orchestra.run.smithi088.stdout:Get:5 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 17.2.7-1jammy [344 kB] 2024-09-13T00:15:59.902 INFO:teuthology.orchestra.run.smithi088.stdout:Get:6 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 17.2.7-1jammy [23.1 kB] 2024-09-13T00:15:59.902 INFO:teuthology.orchestra.run.smithi088.stdout:Get:7 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 17.2.7-1jammy [164 kB] 2024-09-13T00:15:59.903 INFO:teuthology.orchestra.run.smithi088.stdout:Get:8 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 17.2.7-1jammy [47.7 kB] 2024-09-13T00:15:59.904 INFO:teuthology.orchestra.run.smithi088.stdout:Get:9 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 17.2.7-1jammy [313 kB] 2024-09-13T00:15:59.906 INFO:teuthology.orchestra.run.smithi088.stdout:Get:10 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 17.2.7-1jammy [5,086 kB] 2024-09-13T00:15:59.915 INFO:teuthology.orchestra.run.smithi077.stdout:Get:3 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 17.2.7-1jammy [632 kB] 2024-09-13T00:15:59.917 INFO:teuthology.orchestra.run.smithi088.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-09-13T00:15:59.919 INFO:teuthology.orchestra.run.smithi077.stdout:Get:4 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 17.2.7-1jammy [344 kB] 2024-09-13T00:15:59.922 INFO:teuthology.orchestra.run.smithi077.stdout:Get:5 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 17.2.7-1jammy [23.1 kB] 2024-09-13T00:15:59.922 INFO:teuthology.orchestra.run.smithi077.stdout:Get:6 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 17.2.7-1jammy [164 kB] 2024-09-13T00:15:59.923 INFO:teuthology.orchestra.run.smithi077.stdout:Get:7 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 17.2.7-1jammy [47.7 kB] 2024-09-13T00:15:59.924 INFO:teuthology.orchestra.run.smithi077.stdout:Get:8 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 17.2.7-1jammy [313 kB] 2024-09-13T00:15:59.926 INFO:teuthology.orchestra.run.smithi077.stdout:Get:9 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 17.2.7-1jammy [5,086 kB] 2024-09-13T00:15:59.941 INFO:teuthology.orchestra.run.smithi088.stdout:Get:12 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 17.2.7-1jammy [102 kB] 2024-09-13T00:15:59.942 INFO:teuthology.orchestra.run.smithi088.stdout:Get:13 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 17.2.7-1jammy [415 kB] 2024-09-13T00:15:59.945 INFO:teuthology.orchestra.run.smithi088.stdout:Get:14 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 17.2.7-1jammy [25.3 MB] 2024-09-13T00:15:59.953 INFO:teuthology.orchestra.run.smithi088.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-09-13T00:15:59.955 INFO:teuthology.orchestra.run.smithi088.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-09-13T00:15:59.957 INFO:teuthology.orchestra.run.smithi088.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-09-13T00:15:59.959 INFO:teuthology.orchestra.run.smithi077.stdout:Get:10 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 17.2.7-1jammy [102 kB] 2024-09-13T00:15:59.960 INFO:teuthology.orchestra.run.smithi077.stdout:Get:11 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 17.2.7-1jammy [415 kB] 2024-09-13T00:15:59.963 INFO:teuthology.orchestra.run.smithi077.stdout:Get:12 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 17.2.7-1jammy [25.3 MB] 2024-09-13T00:15:59.988 INFO:teuthology.orchestra.run.smithi088.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-09-13T00:15:59.990 INFO:teuthology.orchestra.run.smithi088.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-09-13T00:15:59.995 INFO:teuthology.orchestra.run.smithi088.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-09-13T00:16:00.012 INFO:teuthology.orchestra.run.smithi077.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfmt8 amd64 8.1.1+ds1-2 [60.2 kB] 2024-09-13T00:16:00.034 INFO:teuthology.orchestra.run.smithi088.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-09-13T00:16:00.037 INFO:teuthology.orchestra.run.smithi088.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-09-13T00:16:00.047 INFO:teuthology.orchestra.run.smithi088.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-09-13T00:16:00.050 INFO:teuthology.orchestra.run.smithi088.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-09-13T00:16:00.051 INFO:teuthology.orchestra.run.smithi088.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-09-13T00:16:00.052 INFO:teuthology.orchestra.run.smithi088.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-09-13T00:16:00.052 INFO:teuthology.orchestra.run.smithi088.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-09-13T00:16:00.052 INFO:teuthology.orchestra.run.smithi088.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-09-13T00:16:00.054 INFO:teuthology.orchestra.run.smithi088.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-09-13T00:16:00.054 INFO:teuthology.orchestra.run.smithi088.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-09-13T00:16:00.071 INFO:teuthology.orchestra.run.smithi088.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-09-13T00:16:00.072 INFO:teuthology.orchestra.run.smithi088.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-09-13T00:16:00.072 INFO:teuthology.orchestra.run.smithi088.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-09-13T00:16:00.073 INFO:teuthology.orchestra.run.smithi088.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-09-13T00:16:00.073 INFO:teuthology.orchestra.run.smithi088.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-09-13T00:16:00.075 INFO:teuthology.orchestra.run.smithi088.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-09-13T00:16:00.075 INFO:teuthology.orchestra.run.smithi088.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-09-13T00:16:00.088 INFO:teuthology.orchestra.run.smithi088.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-09-13T00:16:00.089 INFO:teuthology.orchestra.run.smithi088.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-09-13T00:16:00.089 INFO:teuthology.orchestra.run.smithi088.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-09-13T00:16:00.108 INFO:teuthology.orchestra.run.smithi088.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-09-13T00:16:00.108 INFO:teuthology.orchestra.run.smithi088.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2024-09-13T00:16:00.109 INFO:teuthology.orchestra.run.smithi088.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-09-13T00:16:00.110 INFO:teuthology.orchestra.run.smithi088.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-09-13T00:16:00.110 INFO:teuthology.orchestra.run.smithi088.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-09-13T00:16:00.116 INFO:teuthology.orchestra.run.smithi088.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-09-13T00:16:00.116 INFO:teuthology.orchestra.run.smithi088.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-09-13T00:16:00.177 INFO:teuthology.orchestra.run.smithi088.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-09-13T00:16:00.195 INFO:teuthology.orchestra.run.smithi077.stdout:Get:14 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 17.2.7-1jammy [6,378 kB] 2024-09-13T00:16:00.245 INFO:teuthology.orchestra.run.smithi088.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-09-13T00:16:00.246 INFO:teuthology.orchestra.run.smithi088.stdout:Get:50 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 17.2.7-1jammy [6,378 kB] 2024-09-13T00:16:00.247 INFO:teuthology.orchestra.run.smithi088.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-09-13T00:16:00.251 INFO:teuthology.orchestra.run.smithi088.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-idna all 3.3-1ubuntu0.1 [52.1 kB] 2024-09-13T00:16:00.253 INFO:teuthology.orchestra.run.smithi088.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-09-13T00:16:00.256 INFO:teuthology.orchestra.run.smithi088.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-09-13T00:16:00.257 INFO:teuthology.orchestra.run.smithi088.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.2 [181 kB] 2024-09-13T00:16:00.262 INFO:teuthology.orchestra.run.smithi088.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-09-13T00:16:00.292 INFO:teuthology.orchestra.run.smithi088.stdout:Get:57 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 17.2.7-1jammy [224 kB] 2024-09-13T00:16:00.292 INFO:teuthology.orchestra.run.smithi077.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-09-13T00:16:00.293 INFO:teuthology.orchestra.run.smithi088.stdout:Get:58 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 17.2.7-1jammy [115 kB] 2024-09-13T00:16:00.294 INFO:teuthology.orchestra.run.smithi088.stdout:Get:59 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 17.2.7-1jammy [1,429 kB] 2024-09-13T00:16:00.304 INFO:teuthology.orchestra.run.smithi088.stdout:Get:60 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 17.2.7-1jammy [8,201 kB] 2024-09-13T00:16:00.306 INFO:teuthology.orchestra.run.smithi077.stdout:Get:16 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 17.2.7-1jammy [224 kB] 2024-09-13T00:16:00.307 INFO:teuthology.orchestra.run.smithi077.stdout:Get:17 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 17.2.7-1jammy [115 kB] 2024-09-13T00:16:00.308 INFO:teuthology.orchestra.run.smithi077.stdout:Get:18 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 17.2.7-1jammy [1,429 kB] 2024-09-13T00:16:00.318 INFO:teuthology.orchestra.run.smithi077.stdout:Get:19 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 17.2.7-1jammy [8,201 kB] 2024-09-13T00:16:00.340 INFO:teuthology.orchestra.run.smithi088.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-09-13T00:16:00.341 INFO:teuthology.orchestra.run.smithi088.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.2 [108 kB] 2024-09-13T00:16:00.342 INFO:teuthology.orchestra.run.smithi088.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-09-13T00:16:00.342 INFO:teuthology.orchestra.run.smithi088.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-09-13T00:16:00.356 INFO:teuthology.orchestra.run.smithi088.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-09-13T00:16:00.362 INFO:teuthology.orchestra.run.smithi088.stdout:Get:66 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 17.2.7-1jammy [28.4 MB] 2024-09-13T00:16:00.371 INFO:teuthology.orchestra.run.smithi077.stdout:Get:20 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 17.2.7-1jammy [28.4 MB] 2024-09-13T00:16:00.375 INFO:teuthology.orchestra.run.smithi088.stdout:Get:67 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-09-13T00:16:00.377 INFO:teuthology.orchestra.run.smithi088.stdout:Get:68 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-09-13T00:16:00.377 INFO:teuthology.orchestra.run.smithi088.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-09-13T00:16:00.394 INFO:teuthology.orchestra.run.smithi088.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-09-13T00:16:00.394 INFO:teuthology.orchestra.run.smithi088.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-09-13T00:16:00.395 INFO:teuthology.orchestra.run.smithi088.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-09-13T00:16:00.395 INFO:teuthology.orchestra.run.smithi088.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-09-13T00:16:00.396 INFO:teuthology.orchestra.run.smithi088.stdout:Get:74 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-09-13T00:16:00.439 INFO:teuthology.orchestra.run.smithi088.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-09-13T00:16:00.439 INFO:teuthology.orchestra.run.smithi088.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-09-13T00:16:00.439 INFO:teuthology.orchestra.run.smithi088.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-09-13T00:16:00.443 INFO:teuthology.orchestra.run.smithi088.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-09-13T00:16:00.445 INFO:teuthology.orchestra.run.smithi088.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-09-13T00:16:00.446 INFO:teuthology.orchestra.run.smithi088.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-09-13T00:16:00.446 INFO:teuthology.orchestra.run.smithi088.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-09-13T00:16:00.450 INFO:teuthology.orchestra.run.smithi088.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-09-13T00:16:00.456 INFO:teuthology.orchestra.run.smithi088.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2024-09-13T00:16:00.464 INFO:teuthology.orchestra.run.smithi077.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-09-13T00:16:00.470 INFO:teuthology.orchestra.run.smithi088.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-09-13T00:16:00.470 INFO:teuthology.orchestra.run.smithi088.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-09-13T00:16:00.471 INFO:teuthology.orchestra.run.smithi088.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-09-13T00:16:00.471 INFO:teuthology.orchestra.run.smithi088.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-09-13T00:16:00.472 INFO:teuthology.orchestra.run.smithi077.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-09-13T00:16:00.473 INFO:teuthology.orchestra.run.smithi088.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-09-13T00:16:00.474 INFO:teuthology.orchestra.run.smithi088.stdout:Get:89 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-09-13T00:16:00.483 INFO:teuthology.orchestra.run.smithi088.stdout:Get:90 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-09-13T00:16:00.483 INFO:teuthology.orchestra.run.smithi088.stdout:Get:91 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-09-13T00:16:00.483 INFO:teuthology.orchestra.run.smithi088.stdout:Get:92 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-09-13T00:16:00.484 INFO:teuthology.orchestra.run.smithi077.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-09-13T00:16:00.492 INFO:teuthology.orchestra.run.smithi088.stdout:Get:93 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-09-13T00:16:00.492 INFO:teuthology.orchestra.run.smithi088.stdout:Get:94 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-09-13T00:16:00.631 INFO:teuthology.orchestra.run.smithi088.stdout:Get:95 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 17.2.7-1jammy [3,690 B] 2024-09-13T00:16:00.636 INFO:teuthology.orchestra.run.smithi077.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-09-13T00:16:00.700 INFO:teuthology.orchestra.run.smithi088.stdout:Get:96 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 17.2.7-1jammy [783 kB] 2024-09-13T00:16:00.706 INFO:teuthology.orchestra.run.smithi088.stdout:Get:97 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 17.2.7-1jammy [2,120 kB] 2024-09-13T00:16:00.712 INFO:teuthology.orchestra.run.smithi077.stdout:Get:25 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 17.2.7-1jammy [3,690 B] 2024-09-13T00:16:00.712 INFO:teuthology.orchestra.run.smithi077.stdout:Get:26 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 17.2.7-1jammy [783 kB] 2024-09-13T00:16:00.712 INFO:teuthology.orchestra.run.smithi077.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-09-13T00:16:00.717 INFO:teuthology.orchestra.run.smithi077.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-09-13T00:16:00.717 INFO:teuthology.orchestra.run.smithi077.stdout:Get:29 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 17.2.7-1jammy [2,120 kB] 2024-09-13T00:16:00.721 INFO:teuthology.orchestra.run.smithi088.stdout:Get:98 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 17.2.7-1jammy [80.2 kB] 2024-09-13T00:16:00.721 INFO:teuthology.orchestra.run.smithi088.stdout:Get:99 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 17.2.7-1jammy [118 kB] 2024-09-13T00:16:00.722 INFO:teuthology.orchestra.run.smithi088.stdout:Get:100 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 17.2.7-1jammy [1,824 kB] 2024-09-13T00:16:00.733 INFO:teuthology.orchestra.run.smithi077.stdout:Get:30 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 17.2.7-1jammy [80.2 kB] 2024-09-13T00:16:00.736 INFO:teuthology.orchestra.run.smithi077.stdout:Get:31 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 17.2.7-1jammy [118 kB] 2024-09-13T00:16:00.736 INFO:teuthology.orchestra.run.smithi088.stdout:Get:101 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 17.2.7-1jammy [8,615 kB] 2024-09-13T00:16:00.737 INFO:teuthology.orchestra.run.smithi077.stdout:Get:32 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 17.2.7-1jammy [1,824 kB] 2024-09-13T00:16:00.748 INFO:teuthology.orchestra.run.smithi077.stdout:Get:33 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 17.2.7-1jammy [8,615 kB] 2024-09-13T00:16:00.792 INFO:teuthology.orchestra.run.smithi088.stdout:Get:102 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 17.2.7-1jammy [3,814 B] 2024-09-13T00:16:00.792 INFO:teuthology.orchestra.run.smithi088.stdout:Get:103 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 17.2.7-1jammy [52.7 MB] 2024-09-13T00:16:00.804 INFO:teuthology.orchestra.run.smithi077.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-09-13T00:16:00.809 INFO:teuthology.orchestra.run.smithi077.stdout:Get:35 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 17.2.7-1jammy [3,814 B] 2024-09-13T00:16:00.809 INFO:teuthology.orchestra.run.smithi077.stdout:Get:36 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 17.2.7-1jammy [52.7 MB] 2024-09-13T00:16:00.810 INFO:teuthology.orchestra.run.smithi077.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-09-13T00:16:00.832 INFO:teuthology.orchestra.run.smithi077.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-09-13T00:16:00.839 INFO:teuthology.orchestra.run.smithi077.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-09-13T00:16:00.840 INFO:teuthology.orchestra.run.smithi077.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-09-13T00:16:00.841 INFO:teuthology.orchestra.run.smithi077.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-09-13T00:16:00.842 INFO:teuthology.orchestra.run.smithi077.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-09-13T00:16:00.883 INFO:teuthology.orchestra.run.smithi077.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-09-13T00:16:00.885 INFO:teuthology.orchestra.run.smithi077.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-09-13T00:16:00.967 INFO:teuthology.orchestra.run.smithi077.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-09-13T00:16:00.968 INFO:teuthology.orchestra.run.smithi077.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-09-13T00:16:00.968 INFO:teuthology.orchestra.run.smithi077.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-09-13T00:16:00.969 INFO:teuthology.orchestra.run.smithi077.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-09-13T00:16:00.970 INFO:teuthology.orchestra.run.smithi077.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-09-13T00:16:00.971 INFO:teuthology.orchestra.run.smithi077.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-09-13T00:16:00.976 INFO:teuthology.orchestra.run.smithi077.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-09-13T00:16:00.977 INFO:teuthology.orchestra.run.smithi077.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-09-13T00:16:01.049 INFO:teuthology.orchestra.run.smithi077.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-09-13T00:16:01.049 INFO:teuthology.orchestra.run.smithi077.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-09-13T00:16:01.131 INFO:teuthology.orchestra.run.smithi077.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-09-13T00:16:01.131 INFO:teuthology.orchestra.run.smithi077.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-09-13T00:16:01.131 INFO:teuthology.orchestra.run.smithi077.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-webob all 1:1.8.6-1.1ubuntu0.1 [86.7 kB] 2024-09-13T00:16:01.134 INFO:teuthology.orchestra.run.smithi077.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-09-13T00:16:01.135 INFO:teuthology.orchestra.run.smithi077.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-09-13T00:16:01.136 INFO:teuthology.orchestra.run.smithi077.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-09-13T00:16:01.230 INFO:teuthology.orchestra.run.smithi077.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-09-13T00:16:01.230 INFO:teuthology.orchestra.run.smithi077.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-09-13T00:16:01.291 INFO:teuthology.orchestra.run.smithi077.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-09-13T00:16:01.344 INFO:teuthology.orchestra.run.smithi088.stdout:Get:104 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 17.2.7-1jammy [240 kB] 2024-09-13T00:16:01.346 INFO:teuthology.orchestra.run.smithi088.stdout:Get:105 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 17.2.7-1jammy [22.4 kB] 2024-09-13T00:16:01.346 INFO:teuthology.orchestra.run.smithi088.stdout:Get:106 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 17.2.7-1jammy [15.4 MB] 2024-09-13T00:16:01.367 INFO:teuthology.orchestra.run.smithi077.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-09-13T00:16:01.368 INFO:teuthology.orchestra.run.smithi077.stdout:Get:65 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 17.2.7-1jammy [240 kB] 2024-09-13T00:16:01.433 INFO:teuthology.orchestra.run.smithi077.stdout:Get:66 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 17.2.7-1jammy [22.4 kB] 2024-09-13T00:16:01.433 INFO:teuthology.orchestra.run.smithi077.stdout:Get:67 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-09-13T00:16:01.433 INFO:teuthology.orchestra.run.smithi077.stdout:Get:68 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 17.2.7-1jammy [15.4 MB] 2024-09-13T00:16:01.436 INFO:teuthology.orchestra.run.smithi077.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-idna all 3.3-1ubuntu0.1 [52.1 kB] 2024-09-13T00:16:01.438 INFO:teuthology.orchestra.run.smithi077.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-09-13T00:16:01.440 INFO:teuthology.orchestra.run.smithi077.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-09-13T00:16:01.442 INFO:teuthology.orchestra.run.smithi077.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.2 [181 kB] 2024-09-13T00:16:01.447 INFO:teuthology.orchestra.run.smithi077.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-09-13T00:16:01.538 INFO:teuthology.orchestra.run.smithi088.stdout:Get:107 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 17.2.7-1jammy [75.8 kB] 2024-09-13T00:16:01.642 INFO:teuthology.orchestra.run.smithi077.stdout:Get:74 https://chacra.ceph.com/r/ceph/quincy/b12291d110049b2f35e32e0de30d70e9a4c060d2/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 17.2.7-1jammy [75.8 kB] 2024-09-13T00:16:01.642 INFO:teuthology.orchestra.run.smithi077.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-09-13T00:16:01.644 INFO:teuthology.orchestra.run.smithi077.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.2 [108 kB] 2024-09-13T00:16:01.647 INFO:teuthology.orchestra.run.smithi077.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-09-13T00:16:01.647 INFO:teuthology.orchestra.run.smithi077.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-09-13T00:16:01.650 INFO:teuthology.orchestra.run.smithi077.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-09-13T00:16:01.693 INFO:teuthology.orchestra.run.smithi077.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-09-13T00:16:01.697 INFO:teuthology.orchestra.run.smithi077.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-09-13T00:16:01.697 INFO:teuthology.orchestra.run.smithi077.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-09-13T00:16:01.746 INFO:teuthology.orchestra.run.smithi077.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-09-13T00:16:01.746 INFO:teuthology.orchestra.run.smithi077.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-09-13T00:16:01.809 INFO:teuthology.orchestra.run.smithi077.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-09-13T00:16:01.810 INFO:teuthology.orchestra.run.smithi077.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-09-13T00:16:01.811 INFO:teuthology.orchestra.run.smithi077.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-09-13T00:16:01.812 INFO:teuthology.orchestra.run.smithi077.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-09-13T00:16:01.813 INFO:teuthology.orchestra.run.smithi077.stdout:Get:89 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-09-13T00:16:01.814 INFO:teuthology.orchestra.run.smithi077.stdout:Get:90 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-09-13T00:16:01.822 INFO:teuthology.orchestra.run.smithi077.stdout:Get:91 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-09-13T00:16:01.891 INFO:teuthology.orchestra.run.smithi077.stdout:Get:92 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-09-13T00:16:01.894 INFO:teuthology.orchestra.run.smithi077.stdout:Get:93 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-09-13T00:16:01.895 INFO:teuthology.orchestra.run.smithi077.stdout:Get:94 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-09-13T00:16:01.973 INFO:teuthology.orchestra.run.smithi077.stdout:Get:95 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-09-13T00:16:01.979 INFO:teuthology.orchestra.run.smithi077.stdout:Get:96 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.3 [474 kB] 2024-09-13T00:16:01.990 INFO:teuthology.orchestra.run.smithi077.stdout:Get:97 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-09-13T00:16:01.990 INFO:teuthology.orchestra.run.smithi077.stdout:Get:98 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-09-13T00:16:01.992 INFO:teuthology.orchestra.run.smithi077.stdout:Get:99 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-09-13T00:16:01.993 INFO:teuthology.orchestra.run.smithi077.stdout:Get:100 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-09-13T00:16:01.997 INFO:teuthology.orchestra.run.smithi077.stdout:Get:101 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-09-13T00:16:02.054 INFO:teuthology.orchestra.run.smithi077.stdout:Get:102 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-09-13T00:16:02.071 INFO:teuthology.orchestra.run.smithi077.stdout:Get:103 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-09-13T00:16:02.072 INFO:teuthology.orchestra.run.smithi077.stdout:Get:104 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-09-13T00:16:02.105 INFO:teuthology.orchestra.run.smithi088.stdout:Fetched 187 MB in 2s (108 MB/s) 2024-09-13T00:16:02.136 INFO:teuthology.orchestra.run.smithi077.stdout:Get:105 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-09-13T00:16:02.141 INFO:teuthology.orchestra.run.smithi077.stdout:Get:106 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-09-13T00:16:02.142 INFO:teuthology.orchestra.run.smithi077.stdout:Get:107 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-09-13T00:16:02.246 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package libfmt8:amd64. 2024-09-13T00:16:02.341 INFO:teuthology.orchestra.run.smithi088.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165990 files and directories currently installed.) 2024-09-13T00:16:02.347 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../000-libfmt8_8.1.1+ds1-2_amd64.deb ... 2024-09-13T00:16:02.428 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking libfmt8:amd64 (8.1.1+ds1-2) ... 2024-09-13T00:16:02.731 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-09-13T00:16:02.754 INFO:teuthology.orchestra.run.smithi077.stdout:Fetched 187 MB in 2s (79.4 MB/s) 2024-09-13T00:16:02.757 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../001-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-09-13T00:16:02.798 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-13T00:16:02.889 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package libfmt8:amd64. 2024-09-13T00:16:02.986 INFO:teuthology.orchestra.run.smithi077.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165990 files and directories currently installed.) 2024-09-13T00:16:02.991 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../000-libfmt8_8.1.1+ds1-2_amd64.deb ... 2024-09-13T00:16:03.076 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package librados2. 2024-09-13T00:16:03.078 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking libfmt8:amd64 (8.1.1+ds1-2) ... 2024-09-13T00:16:03.102 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../002-librados2_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:03.134 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking librados2 (17.2.7-1jammy) ... 2024-09-13T00:16:03.582 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-09-13T00:16:03.608 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../001-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-09-13T00:16:03.628 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package librbd1. 2024-09-13T00:16:03.648 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-13T00:16:03.654 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../003-librbd1_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:03.695 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking librbd1 (17.2.7-1jammy) ... 2024-09-13T00:16:03.935 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package librados2. 2024-09-13T00:16:03.949 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../002-librados2_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:03.985 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking librados2 (17.2.7-1jammy) ... 2024-09-13T00:16:04.123 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package libcephfs2. 2024-09-13T00:16:04.148 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../004-libcephfs2_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:04.189 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking libcephfs2 (17.2.7-1jammy) ... 2024-09-13T00:16:04.421 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package librbd1. 2024-09-13T00:16:04.447 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../003-librbd1_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:04.458 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-rados. 2024-09-13T00:16:04.473 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../005-python3-rados_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:04.487 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking librbd1 (17.2.7-1jammy) ... 2024-09-13T00:16:04.508 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-rados (17.2.7-1jammy) ... 2024-09-13T00:16:04.761 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-09-13T00:16:04.787 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../006-python3-ceph-argparse_17.2.7-1jammy_all.deb ... 2024-09-13T00:16:04.827 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-ceph-argparse (17.2.7-1jammy) ... 2024-09-13T00:16:04.924 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package libcephfs2. 2024-09-13T00:16:04.949 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../004-libcephfs2_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:04.990 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking libcephfs2 (17.2.7-1jammy) ... 2024-09-13T00:16:05.088 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-cephfs. 2024-09-13T00:16:05.114 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../007-python3-cephfs_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:05.155 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-cephfs (17.2.7-1jammy) ... 2024-09-13T00:16:05.267 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-rados. 2024-09-13T00:16:05.282 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../005-python3-rados_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:05.317 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-rados (17.2.7-1jammy) ... 2024-09-13T00:16:05.491 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-ceph-common. 2024-09-13T00:16:05.517 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../008-python3-ceph-common_17.2.7-1jammy_all.deb ... 2024-09-13T00:16:05.557 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-ceph-common (17.2.7-1jammy) ... 2024-09-13T00:16:05.612 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-09-13T00:16:05.638 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../006-python3-ceph-argparse_17.2.7-1jammy_all.deb ... 2024-09-13T00:16:05.678 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-ceph-argparse (17.2.7-1jammy) ... 2024-09-13T00:16:05.868 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-wcwidth. 2024-09-13T00:16:05.894 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../009-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-09-13T00:16:05.931 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-cephfs. 2024-09-13T00:16:05.935 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-13T00:16:05.953 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../007-python3-cephfs_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:05.998 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-cephfs (17.2.7-1jammy) ... 2024-09-13T00:16:06.230 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-prettytable. 2024-09-13T00:16:06.256 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../010-python3-prettytable_2.5.0-2_all.deb ... 2024-09-13T00:16:06.296 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-09-13T00:16:06.308 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-ceph-common. 2024-09-13T00:16:06.334 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../008-python3-ceph-common_17.2.7-1jammy_all.deb ... 2024-09-13T00:16:06.375 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-ceph-common (17.2.7-1jammy) ... 2024-09-13T00:16:06.574 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-rbd. 2024-09-13T00:16:06.600 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../011-python3-rbd_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:06.640 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-rbd (17.2.7-1jammy) ... 2024-09-13T00:16:06.669 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-wcwidth. 2024-09-13T00:16:06.695 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../009-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-09-13T00:16:06.727 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-13T00:16:07.002 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-09-13T00:16:07.028 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../012-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-09-13T00:16:07.068 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-13T00:16:07.072 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-prettytable. 2024-09-13T00:16:07.098 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../010-python3-prettytable_2.5.0-2_all.deb ... 2024-09-13T00:16:07.147 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-09-13T00:16:07.362 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-09-13T00:16:07.389 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../013-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-09-13T00:16:07.416 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-rbd. 2024-09-13T00:16:07.421 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-13T00:16:07.442 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../011-python3-rbd_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:07.483 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-rbd (17.2.7-1jammy) ... 2024-09-13T00:16:07.749 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-09-13T00:16:07.775 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../014-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-09-13T00:16:07.815 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-13T00:16:07.836 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-09-13T00:16:07.862 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../012-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-09-13T00:16:07.902 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-13T00:16:08.205 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-09-13T00:16:08.231 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../013-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-09-13T00:16:08.252 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-09-13T00:16:08.263 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-13T00:16:08.279 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../015-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-13T00:16:08.326 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T00:16:08.683 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-09-13T00:16:08.697 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-09-13T00:16:08.709 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../014-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-09-13T00:16:08.723 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../016-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-13T00:16:08.749 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-13T00:16:08.755 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T00:16:09.100 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-09-13T00:16:09.126 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../017-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-13T00:16:09.166 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T00:16:09.195 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-09-13T00:16:09.221 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../015-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-13T00:16:09.270 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T00:16:09.519 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-09-13T00:16:09.546 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../018-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-09-13T00:16:09.586 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-13T00:16:09.648 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-09-13T00:16:09.674 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../016-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-13T00:16:09.714 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T00:16:09.864 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package librgw2. 2024-09-13T00:16:09.890 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../019-librgw2_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:09.947 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking librgw2 (17.2.7-1jammy) ... 2024-09-13T00:16:10.067 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-09-13T00:16:10.093 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../017-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-09-13T00:16:10.134 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T00:16:10.383 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-rgw. 2024-09-13T00:16:10.398 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../020-python3-rgw_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:10.441 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-rgw (17.2.7-1jammy) ... 2024-09-13T00:16:10.462 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-09-13T00:16:10.488 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../018-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-09-13T00:16:10.528 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-13T00:16:10.777 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package liboath0:amd64. 2024-09-13T00:16:10.803 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../021-liboath0_2.6.7-3build1_amd64.deb ... 2024-09-13T00:16:10.843 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-09-13T00:16:10.848 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package librgw2. 2024-09-13T00:16:10.874 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../019-librgw2_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:10.914 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking librgw2 (17.2.7-1jammy) ... 2024-09-13T00:16:11.130 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package libradosstriper1. 2024-09-13T00:16:11.156 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../022-libradosstriper1_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:11.196 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking libradosstriper1 (17.2.7-1jammy) ... 2024-09-13T00:16:11.384 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-rgw. 2024-09-13T00:16:11.410 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../020-python3-rgw_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:11.450 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-rgw (17.2.7-1jammy) ... 2024-09-13T00:16:11.566 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package ceph-common. 2024-09-13T00:16:11.593 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../023-ceph-common_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:11.633 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking ceph-common (17.2.7-1jammy) ... 2024-09-13T00:16:11.786 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package liboath0:amd64. 2024-09-13T00:16:11.813 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../021-liboath0_2.6.7-3build1_amd64.deb ... 2024-09-13T00:16:11.853 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-09-13T00:16:12.164 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package libradosstriper1. 2024-09-13T00:16:12.190 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../022-libradosstriper1_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:12.232 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking libradosstriper1 (17.2.7-1jammy) ... 2024-09-13T00:16:12.586 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package ceph-common. 2024-09-13T00:16:12.612 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../023-ceph-common_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:12.652 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking ceph-common (17.2.7-1jammy) ... 2024-09-13T00:16:13.013 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package ceph-base. 2024-09-13T00:16:13.040 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../024-ceph-base_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:13.089 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking ceph-base (17.2.7-1jammy) ... 2024-09-13T00:16:13.751 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-openssl. 2024-09-13T00:16:13.777 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../025-python3-openssl_21.0.0-1_all.deb ... 2024-09-13T00:16:13.817 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-09-13T00:16:14.094 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-09-13T00:16:14.109 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../026-ceph-mgr-modules-core_17.2.7-1jammy_all.deb ... 2024-09-13T00:16:14.153 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-09-13T00:16:14.224 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package ceph-base. 2024-09-13T00:16:14.250 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../024-ceph-base_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:14.300 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking ceph-base (17.2.7-1jammy) ... 2024-09-13T00:16:14.507 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-bcrypt. 2024-09-13T00:16:14.534 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../027-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-09-13T00:16:14.574 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-09-13T00:16:14.877 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-09-13T00:16:14.903 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../028-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-09-13T00:16:14.943 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-09-13T00:16:15.004 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-openssl. 2024-09-13T00:16:15.030 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../025-python3-openssl_21.0.0-1_all.deb ... 2024-09-13T00:16:15.070 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-09-13T00:16:15.255 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-cheroot. 2024-09-13T00:16:15.281 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../029-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-09-13T00:16:15.306 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-09-13T00:16:15.321 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-13T00:16:15.332 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../026-ceph-mgr-modules-core_17.2.7-1jammy_all.deb ... 2024-09-13T00:16:15.364 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-09-13T00:16:15.616 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-09-13T00:16:15.642 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../030-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-09-13T00:16:15.682 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-09-13T00:16:15.744 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-bcrypt. 2024-09-13T00:16:15.770 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../027-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-09-13T00:16:15.810 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-09-13T00:16:15.935 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-jaraco.text. 2024-09-13T00:16:15.961 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../031-python3-jaraco.text_3.6.0-2_all.deb ... 2024-09-13T00:16:16.001 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-09-13T00:16:16.088 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-09-13T00:16:16.114 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../028-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-09-13T00:16:16.155 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-09-13T00:16:16.287 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-09-13T00:16:16.313 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../032-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-09-13T00:16:16.362 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-09-13T00:16:16.424 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-cheroot. 2024-09-13T00:16:16.450 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../029-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-09-13T00:16:16.491 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-13T00:16:16.665 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-tempora. 2024-09-13T00:16:16.691 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../033-python3-tempora_4.1.2-1_all.deb ... 2024-09-13T00:16:16.732 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-09-13T00:16:16.819 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-09-13T00:16:16.845 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../030-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-09-13T00:16:16.885 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-09-13T00:16:16.993 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-portend. 2024-09-13T00:16:17.008 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../034-python3-portend_3.0.0-1_all.deb ... 2024-09-13T00:16:17.042 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-09-13T00:16:17.189 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-jaraco.text. 2024-09-13T00:16:17.214 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../031-python3-jaraco.text_3.6.0-2_all.deb ... 2024-09-13T00:16:17.254 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-09-13T00:16:17.329 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-09-13T00:16:17.355 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../035-python3-zc.lockfile_2.0-1_all.deb ... 2024-09-13T00:16:17.395 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-09-13T00:16:17.532 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-09-13T00:16:17.558 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../032-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-09-13T00:16:17.599 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-09-13T00:16:17.698 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-cherrypy3. 2024-09-13T00:16:17.724 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../036-python3-cherrypy3_18.6.1-4_all.deb ... 2024-09-13T00:16:17.765 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-09-13T00:16:17.877 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-tempora. 2024-09-13T00:16:17.903 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../033-python3-tempora_4.1.2-1_all.deb ... 2024-09-13T00:16:17.943 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-09-13T00:16:18.068 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-natsort. 2024-09-13T00:16:18.083 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../037-python3-natsort_8.0.2-1_all.deb ... 2024-09-13T00:16:18.118 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-09-13T00:16:18.246 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-portend. 2024-09-13T00:16:18.272 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../034-python3-portend_3.0.0-1_all.deb ... 2024-09-13T00:16:18.313 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-09-13T00:16:18.438 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-logutils. 2024-09-13T00:16:18.464 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../038-python3-logutils_0.3.3-8_all.deb ... 2024-09-13T00:16:18.504 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-09-13T00:16:18.657 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-09-13T00:16:18.683 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../035-python3-zc.lockfile_2.0-1_all.deb ... 2024-09-13T00:16:18.724 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-09-13T00:16:19.091 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-markupsafe. 2024-09-13T00:16:19.117 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../039-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-09-13T00:16:19.157 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-09-13T00:16:19.177 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-cherrypy3. 2024-09-13T00:16:19.203 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../036-python3-cherrypy3_18.6.1-4_all.deb ... 2024-09-13T00:16:19.235 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-09-13T00:16:19.443 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-mako. 2024-09-13T00:16:19.470 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../040-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-09-13T00:16:19.509 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-13T00:16:19.588 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-natsort. 2024-09-13T00:16:19.614 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../037-python3-natsort_8.0.2-1_all.deb ... 2024-09-13T00:16:19.655 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-09-13T00:16:19.830 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-simplegeneric. 2024-09-13T00:16:19.856 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../041-python3-simplegeneric_0.8.1-3_all.deb ... 2024-09-13T00:16:19.887 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-09-13T00:16:19.949 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-logutils. 2024-09-13T00:16:19.964 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../038-python3-logutils_0.3.3-8_all.deb ... 2024-09-13T00:16:19.999 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-09-13T00:16:20.149 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-singledispatch. 2024-09-13T00:16:20.175 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../042-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-09-13T00:16:20.215 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-09-13T00:16:20.311 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-markupsafe. 2024-09-13T00:16:20.378 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../039-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-09-13T00:16:20.379 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-09-13T00:16:20.476 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-webob. 2024-09-13T00:16:20.491 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../043-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2024-09-13T00:16:20.526 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-13T00:16:20.664 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-mako. 2024-09-13T00:16:20.690 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../040-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-09-13T00:16:20.721 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-13T00:16:20.871 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-waitress. 2024-09-13T00:16:20.897 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../044-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-09-13T00:16:20.940 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-13T00:16:20.999 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-simplegeneric. 2024-09-13T00:16:21.026 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../041-python3-simplegeneric_0.8.1-3_all.deb ... 2024-09-13T00:16:21.057 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-09-13T00:16:21.224 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-tempita. 2024-09-13T00:16:21.250 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../045-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-09-13T00:16:21.282 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-09-13T00:16:21.385 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-singledispatch. 2024-09-13T00:16:21.412 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../042-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-09-13T00:16:21.452 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-09-13T00:16:21.568 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-paste. 2024-09-13T00:16:21.594 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../046-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-09-13T00:16:21.634 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-09-13T00:16:21.746 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-webob. 2024-09-13T00:16:21.773 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../043-python3-webob_1%3a1.8.6-1.1ubuntu0.1_all.deb ... 2024-09-13T00:16:21.813 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-13T00:16:21.929 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-09-13T00:16:21.944 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../047-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-09-13T00:16:21.979 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-09-13T00:16:22.157 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-waitress. 2024-09-13T00:16:22.184 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../044-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-09-13T00:16:22.227 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-13T00:16:22.249 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-pastedeploy. 2024-09-13T00:16:22.275 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../048-python3-pastedeploy_2.1.1-1_all.deb ... 2024-09-13T00:16:22.315 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-09-13T00:16:22.560 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-tempita. 2024-09-13T00:16:22.586 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../045-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-09-13T00:16:22.618 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-webtest. 2024-09-13T00:16:22.626 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-09-13T00:16:22.644 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../049-python3-webtest_2.0.35-1_all.deb ... 2024-09-13T00:16:22.684 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-09-13T00:16:22.929 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-paste. 2024-09-13T00:16:22.956 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../046-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-09-13T00:16:22.995 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-pecan. 2024-09-13T00:16:22.996 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-09-13T00:16:23.022 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../050-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-09-13T00:16:23.062 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-09-13T00:16:23.257 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-09-13T00:16:23.272 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../047-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-09-13T00:16:23.315 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-09-13T00:16:23.425 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-certifi. 2024-09-13T00:16:23.452 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../051-python3-certifi_2020.6.20-1_all.deb ... 2024-09-13T00:16:23.492 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-09-13T00:16:23.610 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-pastedeploy. 2024-09-13T00:16:23.636 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../048-python3-pastedeploy_2.1.1-1_all.deb ... 2024-09-13T00:16:23.676 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-09-13T00:16:23.786 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-idna. 2024-09-13T00:16:23.812 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../052-python3-idna_3.3-1ubuntu0.1_all.deb ... 2024-09-13T00:16:23.852 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-idna (3.3-1ubuntu0.1) ... 2024-09-13T00:16:23.979 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-webtest. 2024-09-13T00:16:24.006 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../049-python3-webtest_2.0.35-1_all.deb ... 2024-09-13T00:16:24.046 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-09-13T00:16:24.306 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-urllib3. 2024-09-13T00:16:24.332 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../053-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-09-13T00:16:24.372 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-13T00:16:24.490 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-pecan. 2024-09-13T00:16:24.505 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../050-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-09-13T00:16:24.557 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-09-13T00:16:24.675 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-requests. 2024-09-13T00:16:24.702 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../054-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-09-13T00:16:24.750 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-13T00:16:24.895 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-certifi. 2024-09-13T00:16:24.921 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../051-python3-certifi_2020.6.20-1_all.deb ... 2024-09-13T00:16:24.961 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-09-13T00:16:25.061 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-werkzeug. 2024-09-13T00:16:25.088 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../055-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.2_all.deb ... 2024-09-13T00:16:25.128 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-13T00:16:25.289 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-idna. 2024-09-13T00:16:25.316 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../052-python3-idna_3.3-1ubuntu0.1_all.deb ... 2024-09-13T00:16:25.356 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-idna (3.3-1ubuntu0.1) ... 2024-09-13T00:16:25.457 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-09-13T00:16:25.483 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../056-libsqlite3-mod-ceph_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:25.523 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-09-13T00:16:25.675 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-urllib3. 2024-09-13T00:16:25.702 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../053-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-09-13T00:16:25.741 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-13T00:16:25.859 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package ceph-mgr. 2024-09-13T00:16:25.886 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../057-ceph-mgr_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:25.926 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking ceph-mgr (17.2.7-1jammy) ... 2024-09-13T00:16:26.078 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-requests. 2024-09-13T00:16:26.105 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../054-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-09-13T00:16:26.145 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-13T00:16:26.320 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package ceph-mon. 2024-09-13T00:16:26.347 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../058-ceph-mon_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:26.386 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking ceph-mon (17.2.7-1jammy) ... 2024-09-13T00:16:26.456 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-werkzeug. 2024-09-13T00:16:26.471 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../055-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.2_all.deb ... 2024-09-13T00:16:26.506 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-13T00:16:26.834 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-09-13T00:16:26.861 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../056-libsqlite3-mod-ceph_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:26.901 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-09-13T00:16:27.190 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package ceph-osd. 2024-09-13T00:16:27.217 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../059-ceph-osd_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:27.257 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking ceph-osd (17.2.7-1jammy) ... 2024-09-13T00:16:27.270 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package ceph-mgr. 2024-09-13T00:16:27.297 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../057-ceph-mgr_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:27.362 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking ceph-mgr (17.2.7-1jammy) ... 2024-09-13T00:16:27.832 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package ceph-mon. 2024-09-13T00:16:27.858 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../058-ceph-mon_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:27.898 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking ceph-mon (17.2.7-1jammy) ... 2024-09-13T00:16:28.677 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package ceph-osd. 2024-09-13T00:16:28.703 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../059-ceph-osd_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:28.735 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking ceph-osd (17.2.7-1jammy) ... 2024-09-13T00:16:28.851 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package ceph. 2024-09-13T00:16:28.878 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../060-ceph_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:28.918 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking ceph (17.2.7-1jammy) ... 2024-09-13T00:16:29.220 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package ceph-fuse. 2024-09-13T00:16:29.249 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../061-ceph-fuse_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:29.280 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking ceph-fuse (17.2.7-1jammy) ... 2024-09-13T00:16:29.634 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package ceph-mds. 2024-09-13T00:16:29.660 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../062-ceph-mds_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:29.700 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking ceph-mds (17.2.7-1jammy) ... 2024-09-13T00:16:30.087 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package cephadm. 2024-09-13T00:16:30.113 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../063-cephadm_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:30.153 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking cephadm (17.2.7-1jammy) ... 2024-09-13T00:16:30.371 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package ceph. 2024-09-13T00:16:30.386 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../060-ceph_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:30.414 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python-babel-localedata. 2024-09-13T00:16:30.429 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking ceph (17.2.7-1jammy) ... 2024-09-13T00:16:30.441 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../064-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-09-13T00:16:30.481 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-13T00:16:30.749 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package ceph-fuse. 2024-09-13T00:16:30.775 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../061-ceph-fuse_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:30.815 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking ceph-fuse (17.2.7-1jammy) ... 2024-09-13T00:16:31.185 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package ceph-mds. 2024-09-13T00:16:31.212 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../062-ceph-mds_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:31.252 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking ceph-mds (17.2.7-1jammy) ... 2024-09-13T00:16:31.544 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-babel. 2024-09-13T00:16:31.571 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../065-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-09-13T00:16:31.610 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-09-13T00:16:31.705 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package cephadm. 2024-09-13T00:16:31.731 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../063-cephadm_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:31.779 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking cephadm (17.2.7-1jammy) ... 2024-09-13T00:16:31.938 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-jinja2. 2024-09-13T00:16:31.965 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../066-python3-jinja2_3.0.3-1ubuntu0.2_all.deb ... 2024-09-13T00:16:32.005 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-13T00:16:32.065 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python-babel-localedata. 2024-09-13T00:16:32.080 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../064-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-09-13T00:16:32.123 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-13T00:16:32.350 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-09-13T00:16:32.376 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../067-ceph-mgr-cephadm_17.2.7-1jammy_all.deb ... 2024-09-13T00:16:32.416 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-09-13T00:16:32.736 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-repoze.lru. 2024-09-13T00:16:32.751 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../068-python3-repoze.lru_0.7-2_all.deb ... 2024-09-13T00:16:32.786 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-09-13T00:16:33.039 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-routes. 2024-09-13T00:16:33.066 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../069-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-09-13T00:16:33.106 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-09-13T00:16:33.120 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-babel. 2024-09-13T00:16:33.148 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../065-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-09-13T00:16:33.587 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-09-13T00:16:34.168 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-09-13T00:16:34.195 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../070-ceph-mgr-dashboard_17.2.7-1jammy_all.deb ... 2024-09-13T00:16:34.234 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-09-13T00:16:34.299 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-jinja2. 2024-09-13T00:16:34.325 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../066-python3-jinja2_3.0.3-1ubuntu0.2_all.deb ... 2024-09-13T00:16:34.365 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-13T00:16:34.613 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-09-13T00:16:34.628 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../071-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-09-13T00:16:34.662 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-13T00:16:34.910 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-09-13T00:16:34.937 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../067-ceph-mgr-cephadm_17.2.7-1jammy_all.deb ... 2024-09-13T00:16:34.977 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-09-13T00:16:35.133 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-joblib. 2024-09-13T00:16:35.148 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../072-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-09-13T00:16:35.183 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-09-13T00:16:35.289 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-repoze.lru. 2024-09-13T00:16:35.315 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../068-python3-repoze.lru_0.7-2_all.deb ... 2024-09-13T00:16:35.355 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-09-13T00:16:35.545 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-09-13T00:16:35.571 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../073-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-09-13T00:16:35.611 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-09-13T00:16:35.658 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-routes. 2024-09-13T00:16:35.685 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../069-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-09-13T00:16:35.724 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-09-13T00:16:35.897 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-sklearn. 2024-09-13T00:16:35.924 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../074-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-09-13T00:16:35.964 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-13T00:16:36.061 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-09-13T00:16:36.088 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../070-ceph-mgr-dashboard_17.2.7-1jammy_all.deb ... 2024-09-13T00:16:36.127 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-09-13T00:16:36.556 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-09-13T00:16:36.583 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../071-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-09-13T00:16:36.622 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-13T00:16:36.647 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-09-13T00:16:36.674 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../075-ceph-mgr-diskprediction-local_17.2.7-1jammy_all.deb ... 2024-09-13T00:16:36.722 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-09-13T00:16:37.093 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-joblib. 2024-09-13T00:16:37.119 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../072-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-09-13T00:16:37.159 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-09-13T00:16:37.496 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-09-13T00:16:37.523 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../073-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-09-13T00:16:37.562 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-09-13T00:16:37.667 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-cachetools. 2024-09-13T00:16:37.693 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../076-python3-cachetools_5.0.0-1_all.deb ... 2024-09-13T00:16:37.733 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-09-13T00:16:37.890 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-sklearn. 2024-09-13T00:16:37.917 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../074-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-09-13T00:16:37.957 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-13T00:16:37.986 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-pyasn1. 2024-09-13T00:16:38.012 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../077-python3-pyasn1_0.4.8-1_all.deb ... 2024-09-13T00:16:38.044 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-09-13T00:16:38.356 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-09-13T00:16:38.371 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../078-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-09-13T00:16:38.406 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-09-13T00:16:38.683 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-rsa. 2024-09-13T00:16:38.698 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-09-13T00:16:38.710 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../079-python3-rsa_4.8-1_all.deb ... 2024-09-13T00:16:38.726 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../075-ceph-mgr-diskprediction-local_17.2.7-1jammy_all.deb ... 2024-09-13T00:16:38.741 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-rsa (4.8-1) ... 2024-09-13T00:16:38.765 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-09-13T00:16:39.045 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-google-auth. 2024-09-13T00:16:39.072 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../080-python3-google-auth_1.5.1-3_all.deb ... 2024-09-13T00:16:39.111 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-09-13T00:16:39.381 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-09-13T00:16:39.408 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../081-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-09-13T00:16:39.439 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-13T00:16:39.676 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-cachetools. 2024-09-13T00:16:39.703 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../076-python3-cachetools_5.0.0-1_all.deb ... 2024-09-13T00:16:39.751 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-09-13T00:16:40.001 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-websocket. 2024-09-13T00:16:40.030 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../082-python3-websocket_1.2.3-1_all.deb ... 2024-09-13T00:16:40.075 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-09-13T00:16:40.204 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-pyasn1. 2024-09-13T00:16:40.231 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../077-python3-pyasn1_0.4.8-1_all.deb ... 2024-09-13T00:16:40.270 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-09-13T00:16:40.489 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-kubernetes. 2024-09-13T00:16:40.515 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../083-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-09-13T00:16:40.555 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-13T00:16:40.607 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-09-13T00:16:40.634 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../078-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-09-13T00:16:40.674 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-09-13T00:16:40.977 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-rsa. 2024-09-13T00:16:41.003 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../079-python3-rsa_4.8-1_all.deb ... 2024-09-13T00:16:41.043 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-rsa (4.8-1) ... 2024-09-13T00:16:41.219 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-09-13T00:16:41.235 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../084-ceph-mgr-k8sevents_17.2.7-1jammy_all.deb ... 2024-09-13T00:16:41.277 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-09-13T00:16:41.296 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-google-auth. 2024-09-13T00:16:41.312 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../080-python3-google-auth_1.5.1-3_all.deb ... 2024-09-13T00:16:41.346 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-09-13T00:16:41.571 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package libonig5:amd64. 2024-09-13T00:16:41.598 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../085-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-09-13T00:16:41.632 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-09-13T00:16:41.637 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-13T00:16:41.660 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../081-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-09-13T00:16:41.699 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-13T00:16:41.982 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package libjq1:amd64. 2024-09-13T00:16:42.002 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-websocket. 2024-09-13T00:16:42.010 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../086-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-09-13T00:16:42.029 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../082-python3-websocket_1.2.3-1_all.deb ... 2024-09-13T00:16:42.049 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-13T00:16:42.068 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-09-13T00:16:42.326 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package jq. 2024-09-13T00:16:42.342 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../087-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-09-13T00:16:42.384 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-09-13T00:16:42.473 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-kubernetes. 2024-09-13T00:16:42.500 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../083-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-09-13T00:16:42.539 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-13T00:16:42.620 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package socat. 2024-09-13T00:16:42.636 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../088-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-09-13T00:16:42.670 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-09-13T00:16:42.949 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package xmlstarlet. 2024-09-13T00:16:42.976 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../089-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-09-13T00:16:43.015 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-09-13T00:16:43.245 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-09-13T00:16:43.261 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../084-ceph-mgr-k8sevents_17.2.7-1jammy_all.deb ... 2024-09-13T00:16:43.294 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-09-13T00:16:43.360 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package ceph-test. 2024-09-13T00:16:43.376 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../090-ceph-test_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:43.410 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking ceph-test (17.2.7-1jammy) ... 2024-09-13T00:16:43.547 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package libonig5:amd64. 2024-09-13T00:16:43.574 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../085-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-09-13T00:16:43.614 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-13T00:16:43.959 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package libjq1:amd64. 2024-09-13T00:16:43.986 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../086-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-09-13T00:16:44.025 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-13T00:16:44.269 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package jq. 2024-09-13T00:16:44.285 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../087-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-09-13T00:16:44.327 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-09-13T00:16:44.563 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package socat. 2024-09-13T00:16:44.579 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../088-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-09-13T00:16:44.613 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-09-13T00:16:44.866 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package xmlstarlet. 2024-09-13T00:16:44.882 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../089-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-09-13T00:16:44.916 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-09-13T00:16:45.237 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package ceph-test. 2024-09-13T00:16:45.253 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../090-ceph-test_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:45.287 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking ceph-test (17.2.7-1jammy) ... 2024-09-13T00:16:45.798 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package ceph-volume. 2024-09-13T00:16:45.825 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../091-ceph-volume_17.2.7-1jammy_all.deb ... 2024-09-13T00:16:45.855 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking ceph-volume (17.2.7-1jammy) ... 2024-09-13T00:16:46.177 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package libcephfs-dev. 2024-09-13T00:16:46.193 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../092-libcephfs-dev_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:46.235 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking libcephfs-dev (17.2.7-1jammy) ... 2024-09-13T00:16:46.596 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package nvme-cli. 2024-09-13T00:16:46.624 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../093-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2024-09-13T00:16:46.663 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2024-09-13T00:16:47.069 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-iniconfig. 2024-09-13T00:16:47.096 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../094-python3-iniconfig_1.1.1-2_all.deb ... 2024-09-13T00:16:47.135 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-09-13T00:16:47.438 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-pastescript. 2024-09-13T00:16:47.465 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../095-python3-pastescript_2.0.2-4_all.deb ... 2024-09-13T00:16:47.504 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-09-13T00:16:47.707 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package ceph-volume. 2024-09-13T00:16:47.723 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../091-ceph-volume_17.2.7-1jammy_all.deb ... 2024-09-13T00:16:47.765 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking ceph-volume (17.2.7-1jammy) ... 2024-09-13T00:16:47.799 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-pluggy. 2024-09-13T00:16:47.815 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../096-python3-pluggy_0.13.0-7.1_all.deb ... 2024-09-13T00:16:47.849 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-09-13T00:16:48.087 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package libcephfs-dev. 2024-09-13T00:16:48.104 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../092-libcephfs-dev_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:48.135 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-psutil. 2024-09-13T00:16:48.145 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking libcephfs-dev (17.2.7-1jammy) ... 2024-09-13T00:16:48.163 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../097-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-09-13T00:16:48.202 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-09-13T00:16:48.513 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-py. 2024-09-13T00:16:48.523 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package nvme-cli. 2024-09-13T00:16:48.540 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../098-python3-py_1.10.0-1_all.deb ... 2024-09-13T00:16:48.550 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../093-nvme-cli_1.16-3ubuntu0.3_amd64.deb ... 2024-09-13T00:16:48.579 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-py (1.10.0-1) ... 2024-09-13T00:16:48.589 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking nvme-cli (1.16-3ubuntu0.3) ... 2024-09-13T00:16:48.883 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-pygments. 2024-09-13T00:16:48.911 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../099-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-09-13T00:16:48.949 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-09-13T00:16:48.978 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-iniconfig. 2024-09-13T00:16:49.006 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../094-python3-iniconfig_1.1.1-2_all.deb ... 2024-09-13T00:16:49.044 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-09-13T00:16:49.347 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-pastescript. 2024-09-13T00:16:49.375 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../095-python3-pastescript_2.0.2-4_all.deb ... 2024-09-13T00:16:49.414 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-09-13T00:16:49.462 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-pyinotify. 2024-09-13T00:16:49.489 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../100-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-09-13T00:16:49.528 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-09-13T00:16:49.684 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-pluggy. 2024-09-13T00:16:49.700 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../096-python3-pluggy_0.13.0-7.1_all.deb ... 2024-09-13T00:16:49.733 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-09-13T00:16:49.848 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-toml. 2024-09-13T00:16:49.875 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../101-python3-toml_0.10.2-1_all.deb ... 2024-09-13T00:16:49.914 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-09-13T00:16:50.045 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-psutil. 2024-09-13T00:16:50.073 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../097-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-09-13T00:16:50.111 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-09-13T00:16:50.217 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-pytest. 2024-09-13T00:16:50.244 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../102-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-09-13T00:16:50.283 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-09-13T00:16:50.440 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-py. 2024-09-13T00:16:50.466 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../098-python3-py_1.10.0-1_all.deb ... 2024-09-13T00:16:50.506 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-py (1.10.0-1) ... 2024-09-13T00:16:50.637 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package python3-simplejson. 2024-09-13T00:16:50.664 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../103-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-09-13T00:16:50.703 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-09-13T00:16:50.792 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-pygments. 2024-09-13T00:16:50.820 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../099-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-09-13T00:16:50.850 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-09-13T00:16:50.981 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-09-13T00:16:50.996 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../104-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-09-13T00:16:51.039 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-09-13T00:16:51.313 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-pyinotify. 2024-09-13T00:16:51.341 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../100-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-09-13T00:16:51.379 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-09-13T00:16:51.674 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-toml. 2024-09-13T00:16:51.701 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../101-python3-toml_0.10.2-1_all.deb ... 2024-09-13T00:16:51.740 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-09-13T00:16:51.743 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package radosgw. 2024-09-13T00:16:51.770 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../105-radosgw_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:51.809 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking radosgw (17.2.7-1jammy) ... 2024-09-13T00:16:52.043 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-pytest. 2024-09-13T00:16:52.071 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../102-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-09-13T00:16:52.109 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-09-13T00:16:52.463 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package python3-simplejson. 2024-09-13T00:16:52.490 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../103-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-09-13T00:16:52.529 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-09-13T00:16:52.712 INFO:teuthology.orchestra.run.smithi088.stdout:Selecting previously unselected package rbd-fuse. 2024-09-13T00:16:52.728 INFO:teuthology.orchestra.run.smithi088.stdout:Preparing to unpack .../106-rbd-fuse_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:52.770 INFO:teuthology.orchestra.run.smithi088.stdout:Unpacking rbd-fuse (17.2.7-1jammy) ... 2024-09-13T00:16:52.798 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-09-13T00:16:52.814 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../104-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-09-13T00:16:52.856 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-09-13T00:16:53.107 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-09-13T00:16:53.358 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-13T00:16:53.484 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2024-09-13T00:16:53.561 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package radosgw. 2024-09-13T00:16:53.588 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../105-radosgw_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:53.629 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking radosgw (17.2.7-1jammy) ... 2024-09-13T00:16:53.742 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-09-13T00:16:54.058 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-09-13T00:16:54.499 INFO:teuthology.orchestra.run.smithi088.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-09-13T00:16:54.515 INFO:teuthology.orchestra.run.smithi077.stdout:Selecting previously unselected package rbd-fuse. 2024-09-13T00:16:54.531 INFO:teuthology.orchestra.run.smithi077.stdout:Preparing to unpack .../106-rbd-fuse_17.2.7-1jammy_amd64.deb ... 2024-09-13T00:16:54.574 INFO:teuthology.orchestra.run.smithi077.stdout:Unpacking rbd-fuse (17.2.7-1jammy) ... 2024-09-13T00:16:54.878 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up cephadm (17.2.7-1jammy) ... 2024-09-13T00:16:54.993 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-09-13T00:16:55.237 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-09-13T00:16:55.387 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up nvme-cli (1.16-3ubuntu0.3) ... 2024-09-13T00:16:55.671 INFO:teuthology.orchestra.run.smithi088.stdout:Adding system user cephadm....done 2024-09-13T00:16:55.828 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-13T00:16:55.922 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-09-13T00:16:56.102 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-09-13T00:16:56.252 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-09-13T00:16:56.328 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-09-13T00:16:56.580 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-09-13T00:16:56.703 INFO:teuthology.orchestra.run.smithi077.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-09-13T00:16:56.773 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up cephadm (17.2.7-1jammy) ... 2024-09-13T00:16:56.839 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-09-13T00:16:56.965 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-py (1.10.0-1) ... 2024-09-13T00:16:57.292 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-09-13T00:16:57.341 INFO:teuthology.orchestra.run.smithi077.stdout:Adding system user cephadm....done 2024-09-13T00:16:57.512 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-09-13T00:16:57.651 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-09-13T00:16:57.805 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-09-13T00:16:57.902 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-09-13T00:16:58.073 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-09-13T00:16:58.153 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-09-13T00:16:58.341 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-09-13T00:16:58.405 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-09-13T00:16:58.609 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-09-13T00:16:58.664 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-09-13T00:16:58.726 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-py (1.10.0-1) ... 2024-09-13T00:16:58.915 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-ceph-argparse (17.2.7-1jammy) ... 2024-09-13T00:16:59.028 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-09-13T00:16:59.174 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-13T00:16:59.300 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-09-13T00:16:59.363 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-09-13T00:16:59.560 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-13T00:16:59.614 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-09-13T00:16:59.677 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-09-13T00:16:59.890 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-09-13T00:17:00.012 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-13T00:17:00.163 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-09-13T00:17:00.166 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-09-13T00:17:00.431 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-09-13T00:17:00.442 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-09-13T00:17:00.690 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-09-13T00:17:00.718 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-ceph-argparse (17.2.7-1jammy) ... 2024-09-13T00:17:01.008 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-09-13T00:17:01.019 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-09-13T00:17:01.245 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-09-13T00:17:01.513 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-09-13T00:17:01.639 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-09-13T00:17:01.727 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-09-13T00:17:02.003 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-09-13T00:17:02.015 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-09-13T00:17:02.121 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-09-13T00:17:02.141 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-09-13T00:17:02.247 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-13T00:17:02.417 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-09-13T00:17:02.539 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-09-13T00:17:02.702 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-09-13T00:17:02.790 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-09-13T00:17:02.995 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-09-13T00:17:03.167 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-13T00:17:03.460 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-13T00:17:03.680 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-09-13T00:17:03.803 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-09-13T00:17:03.939 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-09-13T00:17:04.037 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-09-13T00:17:04.090 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-09-13T00:17:04.163 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-ceph-common (17.2.7-1jammy) ... 2024-09-13T00:17:04.241 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-09-13T00:17:04.456 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-idna (3.3-1ubuntu0.1) ... 2024-09-13T00:17:04.559 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-09-13T00:17:04.774 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-13T00:17:04.827 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-09-13T00:17:05.228 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-09-13T00:17:05.530 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.2) ... 2024-09-13T00:17:05.864 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-09-13T00:17:05.971 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T00:17:06.105 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-toml (0.10.2-1) ... 2024-09-13T00:17:06.332 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-09-13T00:17:06.365 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-13T00:17:06.483 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-ceph-common (17.2.7-1jammy) ... 2024-09-13T00:17:06.491 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-09-13T00:17:06.617 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-13T00:17:06.784 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-idna (3.3-1ubuntu0.1) ... 2024-09-13T00:17:06.918 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-09-13T00:17:07.102 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-09-13T00:17:07.153 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-09-13T00:17:07.412 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T00:17:07.546 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up libfmt8:amd64 (8.1.1+ds1-2) ... 2024-09-13T00:17:07.673 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-09-13T00:17:07.974 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-09-13T00:17:08.225 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-09-13T00:17:08.266 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T00:17:08.400 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-toml (0.10.2-1) ... 2024-09-13T00:17:08.492 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-09-13T00:17:08.602 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-09-13T00:17:08.651 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-09-13T00:17:08.769 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-09-13T00:17:08.853 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-09-13T00:17:08.878 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-09-13T00:17:09.112 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-09-13T00:17:09.188 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-09-13T00:17:09.380 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-13T00:17:09.439 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-09-13T00:17:09.506 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-09-13T00:17:09.707 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T00:17:09.816 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-13T00:17:09.858 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up libfmt8:amd64 (8.1.1+ds1-2) ... 2024-09-13T00:17:09.942 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-13T00:17:10.009 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-09-13T00:17:10.226 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-13T00:17:10.318 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-09-13T00:17:10.519 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-09-13T00:17:10.569 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-09-13T00:17:10.762 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up librados2 (17.2.7-1jammy) ... 2024-09-13T00:17:10.854 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up ceph-mgr-modules-core (17.2.7-1jammy) ... 2024-09-13T00:17:10.913 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-09-13T00:17:10.980 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-09-13T00:17:11.248 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-09-13T00:17:11.311 INFO:teuthology.orchestra.run.smithi088.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-09-13T00:17:11.524 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-09-13T00:17:11.557 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-09-13T00:17:11.683 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-13T00:17:11.809 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-09-13T00:17:11.825 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-09-13T00:17:11.976 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-09-13T00:17:12.194 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-09-13T00:17:12.302 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-09-13T00:17:12.411 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-09-13T00:17:12.454 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-portend (3.0.0-1) ... 2024-09-13T00:17:12.705 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T00:17:12.713 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-webob (1:1.8.6-1.1ubuntu0.1) ... 2024-09-13T00:17:12.831 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up libcephfs2 (17.2.7-1jammy) ... 2024-09-13T00:17:12.956 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-13T00:17:13.022 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-09-13T00:17:13.273 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up librados2 (17.2.7-1jammy) ... 2024-09-13T00:17:13.274 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up libradosstriper1 (17.2.7-1jammy) ... 2024-09-13T00:17:13.400 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-09-13T00:17:13.424 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-09-13T00:17:13.693 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-13T00:17:13.849 INFO:teuthology.orchestra.run.smithi077.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-09-13T00:17:13.952 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up librbd1 (17.2.7-1jammy) ... 2024-09-13T00:17:14.079 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-09-13T00:17:14.110 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up libsqlite3-mod-ceph (17.2.7-1jammy) ... 2024-09-13T00:17:14.204 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-rsa (4.8-1) ... 2024-09-13T00:17:14.261 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-09-13T00:17:14.412 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-09-13T00:17:14.472 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-09-13T00:17:14.748 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up ceph-fuse (17.2.7-1jammy) ... 2024-09-13T00:17:14.822 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-09-13T00:17:14.963 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-13T00:17:14.963 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-13T00:17:15.090 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-portend (3.0.0-1) ... 2024-09-13T00:17:15.366 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-09-13T00:17:15.441 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up libcephfs-dev (17.2.7-1jammy) ... 2024-09-13T00:17:15.517 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up libcephfs2 (17.2.7-1jammy) ... 2024-09-13T00:17:15.567 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-13T00:17:15.668 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.2) ... 2024-09-13T00:17:15.835 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-09-13T00:17:16.019 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up libradosstriper1 (17.2.7-1jammy) ... 2024-09-13T00:17:16.145 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-09-13T00:17:16.230 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-09-13T00:17:16.463 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-09-13T00:17:16.523 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-09-13T00:17:16.739 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up librbd1 (17.2.7-1jammy) ... 2024-09-13T00:17:16.865 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-09-13T00:17:16.944 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-13T00:17:17.008 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-rsa (4.8-1) ... 2024-09-13T00:17:17.070 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-rados (17.2.7-1jammy) ... 2024-09-13T00:17:17.197 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up librgw2 (17.2.7-1jammy) ... 2024-09-13T00:17:17.300 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-09-13T00:17:17.323 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-rbd (17.2.7-1jammy) ... 2024-09-13T00:17:17.449 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up rbd-fuse (17.2.7-1jammy) ... 2024-09-13T00:17:17.566 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-09-13T00:17:17.568 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up ceph-fuse (17.2.7-1jammy) ... 2024-09-13T00:17:17.798 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-13T00:17:17.798 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-09-13T00:17:17.818 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-rgw (17.2.7-1jammy) ... 2024-09-13T00:17:17.927 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-cephfs (17.2.7-1jammy) ... 2024-09-13T00:17:18.078 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-13T00:17:18.303 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up libcephfs-dev (17.2.7-1jammy) ... 2024-09-13T00:17:18.429 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-09-13T00:17:18.713 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-09-13T00:17:19.106 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-09-13T00:17:19.307 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up ceph-common (17.2.7-1jammy) ... 2024-09-13T00:17:19.399 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-09-13T00:17:19.800 INFO:teuthology.orchestra.run.smithi088.stdout:Adding group ceph....done 2024-09-13T00:17:19.820 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-09-13T00:17:19.980 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-rados (17.2.7-1jammy) ... 2024-09-13T00:17:20.106 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up librgw2 (17.2.7-1jammy) ... 2024-09-13T00:17:20.232 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-rbd (17.2.7-1jammy) ... 2024-09-13T00:17:20.252 INFO:teuthology.orchestra.run.smithi088.stdout:Adding system user ceph....done 2024-09-13T00:17:20.395 INFO:teuthology.orchestra.run.smithi088.stdout:Setting system user ceph properties....done 2024-09-13T00:17:20.408 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up rbd-fuse (17.2.7-1jammy) ... 2024-09-13T00:17:20.412 INFO:teuthology.orchestra.run.smithi088.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-09-13T00:17:20.542 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-09-13T00:17:20.559 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-09-13T00:17:20.844 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-09-13T00:17:20.860 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-rgw (17.2.7-1jammy) ... 2024-09-13T00:17:20.978 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-cephfs (17.2.7-1jammy) ... 2024-09-13T00:17:21.104 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-09-13T00:17:21.325 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up ceph-test (17.2.7-1jammy) ... 2024-09-13T00:17:21.451 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up radosgw (17.2.7-1jammy) ... 2024-09-13T00:17:21.712 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-13T00:17:21.712 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-13T00:17:22.324 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up ceph-common (17.2.7-1jammy) ... 2024-09-13T00:17:22.428 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up ceph-base (17.2.7-1jammy) ... 2024-09-13T00:17:22.776 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-09-13T00:17:22.942 INFO:teuthology.orchestra.run.smithi077.stdout:Adding group ceph....done 2024-09-13T00:17:23.247 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up ceph-mds (17.2.7-1jammy) ... 2024-09-13T00:17:23.377 INFO:teuthology.orchestra.run.smithi077.stdout:Adding system user ceph....done 2024-09-13T00:17:23.450 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-13T00:17:23.450 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-13T00:17:23.588 INFO:teuthology.orchestra.run.smithi077.stdout:Setting system user ceph properties....done 2024-09-13T00:17:23.607 INFO:teuthology.orchestra.run.smithi077.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-09-13T00:17:23.738 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-09-13T00:17:23.907 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up ceph-mgr (17.2.7-1jammy) ... 2024-09-13T00:17:24.068 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-09-13T00:17:24.104 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-13T00:17:24.104 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-13T00:17:24.551 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up ceph-test (17.2.7-1jammy) ... 2024-09-13T00:17:24.583 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up ceph-osd (17.2.7-1jammy) ... 2024-09-13T00:17:24.702 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up radosgw (17.2.7-1jammy) ... 2024-09-13T00:17:24.847 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-13T00:17:24.847 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-13T00:17:24.988 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-13T00:17:24.988 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-09-13T00:17:25.319 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-09-13T00:17:25.445 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-09-13T00:17:25.604 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up ceph-mon (17.2.7-1jammy) ... 2024-09-13T00:17:25.687 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up ceph-base (17.2.7-1jammy) ... 2024-09-13T00:17:25.816 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-13T00:17:25.816 INFO:teuthology.orchestra.run.smithi088.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-13T00:17:26.055 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-09-13T00:17:26.514 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up ceph-mds (17.2.7-1jammy) ... 2024-09-13T00:17:26.575 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-09-13T00:17:26.733 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-13T00:17:26.733 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-09-13T00:17:26.910 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up ceph (17.2.7-1jammy) ... 2024-09-13T00:17:27.027 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-09-13T00:17:27.178 INFO:teuthology.orchestra.run.smithi088.stdout:Setting up ceph-volume (17.2.7-1jammy) ... 2024-09-13T00:17:27.216 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up ceph-mgr (17.2.7-1jammy) ... 2024-09-13T00:17:27.314 INFO:teuthology.orchestra.run.smithi088.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-09-13T00:17:27.452 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-13T00:17:27.452 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-09-13T00:17:27.975 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up ceph-osd (17.2.7-1jammy) ... 2024-09-13T00:17:28.228 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-13T00:17:28.228 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-09-13T00:17:28.440 INFO:teuthology.orchestra.run.smithi088.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-09-13T00:17:28.677 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up ceph-mgr-k8sevents (17.2.7-1jammy) ... 2024-09-13T00:17:28.811 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up ceph-mgr-diskprediction-local (17.2.7-1jammy) ... 2024-09-13T00:17:28.907 INFO:teuthology.orchestra.run.smithi088.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-09-13T00:17:28.915 DEBUG:teuthology.parallel:result is None 2024-09-13T00:17:28.953 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up ceph-mon (17.2.7-1jammy) ... 2024-09-13T00:17:29.184 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-13T00:17:29.184 INFO:teuthology.orchestra.run.smithi077.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-09-13T00:17:29.664 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up ceph-mgr-cephadm (17.2.7-1jammy) ... 2024-09-13T00:17:29.814 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up ceph (17.2.7-1jammy) ... 2024-09-13T00:17:29.940 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up ceph-mgr-dashboard (17.2.7-1jammy) ... 2024-09-13T00:17:30.091 INFO:teuthology.orchestra.run.smithi077.stdout:Setting up ceph-volume (17.2.7-1jammy) ... 2024-09-13T00:17:30.210 INFO:teuthology.orchestra.run.smithi077.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-09-13T00:17:31.463 INFO:teuthology.orchestra.run.smithi077.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-09-13T00:17:31.945 INFO:teuthology.orchestra.run.smithi077.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-09-13T00:17:31.951 DEBUG:teuthology.parallel:result is None 2024-09-13T00:17:31.951 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T00:17:31.951 INFO:teuthology.packaging:ref: None 2024-09-13T00:17:31.951 INFO:teuthology.packaging:tag: None 2024-09-13T00:17:31.951 INFO:teuthology.packaging:branch: quincy 2024-09-13T00:17:31.951 INFO:teuthology.packaging:sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-13T00:17:31.952 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-09-13T00:17:32.099 DEBUG:teuthology.orchestra.run.smithi077:> dpkg-query -W -f '${Version}' ceph 2024-09-13T00:17:32.118 INFO:teuthology.orchestra.run.smithi077.stdout:17.2.7-1jammy 2024-09-13T00:17:32.119 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1jammy 2024-09-13T00:17:32.119 INFO:teuthology.task.install:The correct ceph version 17.2.7-1jammy is installed. 2024-09-13T00:17:32.121 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-13T00:17:32.121 INFO:teuthology.packaging:ref: None 2024-09-13T00:17:32.121 INFO:teuthology.packaging:tag: None 2024-09-13T00:17:32.121 INFO:teuthology.packaging:branch: quincy 2024-09-13T00:17:32.121 INFO:teuthology.packaging:sha1: 7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-13T00:17:32.121 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=quincy 2024-09-13T00:17:32.259 DEBUG:teuthology.orchestra.run.smithi088:> dpkg-query -W -f '${Version}' ceph 2024-09-13T00:17:32.278 INFO:teuthology.orchestra.run.smithi088.stdout:17.2.7-1jammy 2024-09-13T00:17:32.278 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-1jammy 2024-09-13T00:17:32.278 INFO:teuthology.task.install:The correct ceph version 17.2.7-1jammy is installed. 2024-09-13T00:17:32.280 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-13T00:17:32.280 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:17:32.280 DEBUG:teuthology.orchestra.run.smithi077:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-13T00:17:32.294 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-13T00:17:32.294 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-13T00:17:32.333 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-13T00:17:32.334 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:17:32.334 DEBUG:teuthology.orchestra.run.smithi077:> sudo dd of=/usr/bin/daemon-helper 2024-09-13T00:17:32.349 DEBUG:teuthology.orchestra.run.smithi077:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-13T00:17:32.404 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-13T00:17:32.404 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd of=/usr/bin/daemon-helper 2024-09-13T00:17:32.417 DEBUG:teuthology.orchestra.run.smithi088:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-13T00:17:32.472 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-13T00:17:32.473 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:17:32.473 DEBUG:teuthology.orchestra.run.smithi077:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-13T00:17:32.486 DEBUG:teuthology.orchestra.run.smithi077:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-13T00:17:32.540 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-13T00:17:32.541 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-13T00:17:32.554 DEBUG:teuthology.orchestra.run.smithi088:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-13T00:17:32.608 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-13T00:17:32.609 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:17:32.609 DEBUG:teuthology.orchestra.run.smithi077:> sudo dd of=/usr/bin/stdin-killer 2024-09-13T00:17:32.623 DEBUG:teuthology.orchestra.run.smithi077:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-13T00:17:32.676 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-13T00:17:32.676 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd of=/usr/bin/stdin-killer 2024-09-13T00:17:32.690 DEBUG:teuthology.orchestra.run.smithi088:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-13T00:17:32.744 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-13T00:17:32.846 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '7a853b1706d076d58e0da7a5d3b9b9eb36276880'} 2024-09-13T00:17:32.846 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-09-13T00:17:32.847 INFO:tasks.cephadm:Cluster fsid is 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:17:32.847 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-13T00:17:32.847 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.77', 'mon.c': '[v2:172.21.15.77:3301,v1:172.21.15.77:6790]', 'mon.b': '172.21.15.88'} 2024-09-13T00:17:32.847 INFO:tasks.cephadm:First mon is mon.a on smithi077 2024-09-13T00:17:32.847 INFO:tasks.cephadm:First mgr is y 2024-09-13T00:17:32.847 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-13T00:17:32.847 DEBUG:teuthology.orchestra.run.smithi077:> sudo hostname $(hostname -s) 2024-09-13T00:17:32.862 DEBUG:teuthology.orchestra.run.smithi088:> sudo hostname $(hostname -s) 2024-09-13T00:17:32.877 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-09-13T00:17:32.877 DEBUG:teuthology.orchestra.run.smithi077:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T00:17:33.130 INFO:teuthology.orchestra.run.smithi077.stdout:-rw-rw-r-- 1 ubuntu ubuntu 378585 Sep 13 00:17 /home/ubuntu/cephtest/cephadm 2024-09-13T00:17:33.131 DEBUG:teuthology.orchestra.run.smithi088:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-13T00:17:33.369 INFO:teuthology.orchestra.run.smithi088.stdout:-rw-rw-r-- 1 ubuntu ubuntu 378585 Sep 13 00:17 /home/ubuntu/cephtest/cephadm 2024-09-13T00:17:33.369 DEBUG:teuthology.orchestra.run.smithi077:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T00:17:33.377 DEBUG:teuthology.orchestra.run.smithi088:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-13T00:17:33.398 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-09-13T00:17:33.398 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-13T00:17:33.423 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-09-13T00:17:33.589 INFO:teuthology.orchestra.run.smithi077.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T00:17:33.593 INFO:teuthology.orchestra.run.smithi088.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T00:18:40.908 INFO:teuthology.orchestra.run.smithi088.stdout:{ 2024-09-13T00:18:40.909 INFO:teuthology.orchestra.run.smithi088.stdout: "ceph_version": "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)", 2024-09-13T00:18:40.909 INFO:teuthology.orchestra.run.smithi088.stdout: "image_id": "214985049bdbd91e1f28ec4bd8bf7dfb097e67a214bb40938f958f4a20cc6f0d", 2024-09-13T00:18:40.909 INFO:teuthology.orchestra.run.smithi088.stdout: "repo_digests": [ 2024-09-13T00:18:40.909 INFO:teuthology.orchestra.run.smithi088.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:9e84e422c5f3e13e9ae13013f0985fd8f54e85e87f1d2d371f86d595f20a9905" 2024-09-13T00:18:40.909 INFO:teuthology.orchestra.run.smithi088.stdout: ] 2024-09-13T00:18:40.909 INFO:teuthology.orchestra.run.smithi088.stdout:} 2024-09-13T00:18:45.938 INFO:teuthology.orchestra.run.smithi077.stdout:{ 2024-09-13T00:18:45.938 INFO:teuthology.orchestra.run.smithi077.stdout: "ceph_version": "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)", 2024-09-13T00:18:45.938 INFO:teuthology.orchestra.run.smithi077.stdout: "image_id": "214985049bdbd91e1f28ec4bd8bf7dfb097e67a214bb40938f958f4a20cc6f0d", 2024-09-13T00:18:45.938 INFO:teuthology.orchestra.run.smithi077.stdout: "repo_digests": [ 2024-09-13T00:18:45.938 INFO:teuthology.orchestra.run.smithi077.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:9e84e422c5f3e13e9ae13013f0985fd8f54e85e87f1d2d371f86d595f20a9905" 2024-09-13T00:18:45.938 INFO:teuthology.orchestra.run.smithi077.stdout: ] 2024-09-13T00:18:45.938 INFO:teuthology.orchestra.run.smithi077.stdout:} 2024-09-13T00:18:45.977 DEBUG:teuthology.orchestra.run.smithi077:> sudo mkdir -p /etc/ceph 2024-09-13T00:18:45.991 DEBUG:teuthology.orchestra.run.smithi088:> sudo mkdir -p /etc/ceph 2024-09-13T00:18:46.005 DEBUG:teuthology.orchestra.run.smithi077:> sudo chmod 777 /etc/ceph 2024-09-13T00:18:46.049 DEBUG:teuthology.orchestra.run.smithi088:> sudo chmod 777 /etc/ceph 2024-09-13T00:18:46.061 INFO:tasks.cephadm:Writing seed config... 2024-09-13T00:18:46.062 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-09-13T00:18:46.063 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-09-13T00:18:46.063 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-13T00:18:46.063 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-13T00:18:46.063 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-13T00:18:46.063 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-09-13T00:18:46.063 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-13T00:18:46.063 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-13T00:18:46.063 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-13T00:18:46.063 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-13T00:18:46.063 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-13T00:18:46.064 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:18:46.064 DEBUG:teuthology.orchestra.run.smithi077:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-13T00:18:46.101 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 9218306c-7165-11ef-bceb-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-13T00:18:46.101 DEBUG:teuthology.orchestra.run.smithi077:mon.a> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.a.service 2024-09-13T00:18:46.147 DEBUG:teuthology.orchestra.run.smithi077:mgr.y> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@mgr.y.service 2024-09-13T00:18:46.193 INFO:tasks.cephadm:Bootstrapping... 2024-09-13T00:18:46.193 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 9218306c-7165-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.77 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-13T00:18:46.406 INFO:teuthology.orchestra.run.smithi077.stdout:-------------------------------------------------------------------------------- 2024-09-13T00:18:46.406 INFO:teuthology.orchestra.run.smithi077.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '9218306c-7165-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.77', '--skip-admin-label'] 2024-09-13T00:18:46.406 INFO:teuthology.orchestra.run.smithi077.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-09-13T00:18:46.407 INFO:teuthology.orchestra.run.smithi077.stdout:Verifying podman|docker is present... 2024-09-13T00:18:46.407 INFO:teuthology.orchestra.run.smithi077.stdout:Verifying lvm2 is present... 2024-09-13T00:18:46.407 INFO:teuthology.orchestra.run.smithi077.stdout:Verifying time synchronization is in place... 2024-09-13T00:18:46.412 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-13T00:18:46.413 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T00:18:46.417 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-13T00:18:46.417 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stdout inactive 2024-09-13T00:18:46.422 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2024-09-13T00:18:46.422 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2024-09-13T00:18:46.425 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2024-09-13T00:18:46.425 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stdout inactive 2024-09-13T00:18:46.431 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2024-09-13T00:18:46.431 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stdout masked 2024-09-13T00:18:46.435 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2024-09-13T00:18:46.435 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stdout inactive 2024-09-13T00:18:46.439 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2024-09-13T00:18:46.440 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2024-09-13T00:18:46.444 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2024-09-13T00:18:46.444 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stdout inactive 2024-09-13T00:18:46.449 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stdout enabled 2024-09-13T00:18:46.453 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stdout active 2024-09-13T00:18:46.453 INFO:teuthology.orchestra.run.smithi077.stdout:Unit ntp.service is enabled and running 2024-09-13T00:18:46.453 INFO:teuthology.orchestra.run.smithi077.stdout:Repeating the final host check... 2024-09-13T00:18:46.453 INFO:teuthology.orchestra.run.smithi077.stdout:docker (/usr/bin/docker) is present 2024-09-13T00:18:46.453 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl is present 2024-09-13T00:18:46.454 INFO:teuthology.orchestra.run.smithi077.stdout:lvcreate is present 2024-09-13T00:18:46.457 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-13T00:18:46.457 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-13T00:18:46.460 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-13T00:18:46.460 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stdout inactive 2024-09-13T00:18:46.464 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2024-09-13T00:18:46.464 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2024-09-13T00:18:46.468 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2024-09-13T00:18:46.468 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stdout inactive 2024-09-13T00:18:46.472 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2024-09-13T00:18:46.472 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stdout masked 2024-09-13T00:18:46.475 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2024-09-13T00:18:46.475 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stdout inactive 2024-09-13T00:18:46.479 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2024-09-13T00:18:46.479 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2024-09-13T00:18:46.483 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2024-09-13T00:18:46.483 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stdout inactive 2024-09-13T00:18:46.487 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stdout enabled 2024-09-13T00:18:46.491 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stdout active 2024-09-13T00:18:46.491 INFO:teuthology.orchestra.run.smithi077.stdout:Unit ntp.service is enabled and running 2024-09-13T00:18:46.491 INFO:teuthology.orchestra.run.smithi077.stdout:Host looks OK 2024-09-13T00:18:46.491 INFO:teuthology.orchestra.run.smithi077.stdout:Cluster fsid: 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:18:46.491 INFO:teuthology.orchestra.run.smithi077.stdout:Acquiring lock 140230026610000 on /run/cephadm/9218306c-7165-11ef-bceb-c7b262605968.lock 2024-09-13T00:18:46.491 INFO:teuthology.orchestra.run.smithi077.stdout:Lock 140230026610000 acquired on /run/cephadm/9218306c-7165-11ef-bceb-c7b262605968.lock 2024-09-13T00:18:46.491 INFO:teuthology.orchestra.run.smithi077.stdout:Verifying IP 172.21.15.77 port 3300 ... 2024-09-13T00:18:46.491 INFO:teuthology.orchestra.run.smithi077.stdout:Verifying IP 172.21.15.77 port 6789 ... 2024-09-13T00:18:46.492 INFO:teuthology.orchestra.run.smithi077.stdout:Base mon IP(s) is [172.21.15.77:3300, 172.21.15.77:6789], mon addrv is [v2:172.21.15.77:3300,v1:172.21.15.77:6789] 2024-09-13T00:18:46.497 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 2024-09-13T00:18:46.497 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.77 metric 100 2024-09-13T00:18:46.497 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout 158.69.67.47 via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.77 metric 100 2024-09-13T00:18:46.498 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2024-09-13T00:18:46.498 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.77 metric 100 2024-09-13T00:18:46.498 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout 172.21.0.1 dev enp3s0f1 proto dhcp scope link src 172.21.15.77 metric 100 2024-09-13T00:18:46.498 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout 172.21.0.2 dev enp3s0f1 proto dhcp scope link src 172.21.15.77 metric 100 2024-09-13T00:18:46.498 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout 172.21.15.254 dev enp3s0f1 proto dhcp scope link src 172.21.15.77 metric 100 2024-09-13T00:18:46.498 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-09-13T00:18:46.498 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-09-13T00:18:46.498 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 expires 1799sec pref medium 2024-09-13T00:18:46.500 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-13T00:18:46.500 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout inet6 ::1/128 scope host 2024-09-13T00:18:46.500 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-13T00:18:46.500 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-09-13T00:18:46.500 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout inet6 fe80::ec4:7aff:fe8f:d3d3/64 scope link 2024-09-13T00:18:46.500 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-13T00:18:46.500 INFO:teuthology.orchestra.run.smithi077.stdout:Mon IP `172.21.15.77` is in CIDR network `172.21.0.0/20` 2024-09-13T00:18:46.500 INFO:teuthology.orchestra.run.smithi077.stdout:Mon IP `172.21.15.77` is in CIDR network `172.21.0.0/20` 2024-09-13T00:18:46.500 INFO:teuthology.orchestra.run.smithi077.stdout:Mon IP `172.21.15.77` is in CIDR network `172.21.0.1/32` 2024-09-13T00:18:46.501 INFO:teuthology.orchestra.run.smithi077.stdout:Mon IP `172.21.15.77` is in CIDR network `172.21.0.1/32` 2024-09-13T00:18:46.501 INFO:teuthology.orchestra.run.smithi077.stdout:Mon IP `172.21.15.77` is in CIDR network `172.21.0.2/32` 2024-09-13T00:18:46.501 INFO:teuthology.orchestra.run.smithi077.stdout:Mon IP `172.21.15.77` is in CIDR network `172.21.0.2/32` 2024-09-13T00:18:46.501 INFO:teuthology.orchestra.run.smithi077.stdout:Mon IP `172.21.15.77` is in CIDR network `172.21.15.254/32` 2024-09-13T00:18:46.501 INFO:teuthology.orchestra.run.smithi077.stdout:Mon IP `172.21.15.77` is in CIDR network `172.21.15.254/32` 2024-09-13T00:18:46.501 INFO:teuthology.orchestra.run.smithi077.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20', '172.21.0.1/32', '172.21.0.1/32', '172.21.0.2/32', '172.21.0.2/32', '172.21.15.254/32', '172.21.15.254/32'] 2024-09-13T00:18:46.501 INFO:teuthology.orchestra.run.smithi077.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-13T00:18:46.501 INFO:teuthology.orchestra.run.smithi077.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-09-13T00:18:46.760 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/docker: stdout quincy: Pulling from ceph-ci/ceph 2024-09-13T00:18:46.761 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/docker: stdout Digest: sha256:9e84e422c5f3e13e9ae13013f0985fd8f54e85e87f1d2d371f86d595f20a9905 2024-09-13T00:18:46.761 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/docker: stdout Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:quincy 2024-09-13T00:18:46.761 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/docker: stdout quay.ceph.io/ceph-ci/ceph:quincy 2024-09-13T00:18:48.251 INFO:teuthology.orchestra.run.smithi077.stdout:ceph: stdout ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable) 2024-09-13T00:18:48.251 INFO:teuthology.orchestra.run.smithi077.stdout:Ceph version: ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable) 2024-09-13T00:18:48.251 INFO:teuthology.orchestra.run.smithi077.stdout:Extracting ceph user uid/gid from container image... 2024-09-13T00:18:49.590 INFO:teuthology.orchestra.run.smithi077.stdout:stat: stdout 167 167 2024-09-13T00:18:49.591 INFO:teuthology.orchestra.run.smithi077.stdout:Creating initial keys... 2024-09-13T00:18:50.938 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-authtool: stdout AQDqhONmSMcAGxAAva2w34hn7fzEsh9qLpy7AA== 2024-09-13T00:18:52.244 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-authtool: stdout AQDrhONmw7MJLxAAspIPvQ0oAmFga0WqfKBbkA== 2024-09-13T00:18:53.592 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-authtool: stdout AQDthONmL6XOAhAAg861Wf8RBa1Ag0CAMRU+mA== 2024-09-13T00:18:53.593 INFO:teuthology.orchestra.run.smithi077.stdout:Creating initial monmap... 2024-09-13T00:18:54.967 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T00:18:54.967 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-09-13T00:18:54.967 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:18:54.967 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T00:18:54.967 INFO:teuthology.orchestra.run.smithi077.stdout:monmaptool for a [v2:172.21.15.77:3300,v1:172.21.15.77:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-13T00:18:54.967 INFO:teuthology.orchestra.run.smithi077.stdout:setting min_mon_release = octopus 2024-09-13T00:18:54.967 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/monmaptool: set fsid to 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:18:54.967 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-13T00:18:54.967 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:18:54.967 INFO:teuthology.orchestra.run.smithi077.stdout:Creating mon... 2024-09-13T00:18:56.418 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.893+0000 7ffb5bb40900 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-13T00:18:56.418 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.893+0000 7ffb5bb40900 1 imported monmap: 2024-09-13T00:18:56.418 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr epoch 0 2024-09-13T00:18:56.418 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr fsid 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:18:56.418 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr last_changed 2024-09-13T00:18:54.475459+0000 2024-09-13T00:18:56.418 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr created 2024-09-13T00:18:54.475459+0000 2024-09-13T00:18:56.418 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr min_mon_release 15 (octopus) 2024-09-13T00:18:56.418 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr election_strategy: 1 2024-09-13T00:18:56.418 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 0: [v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0] mon.a 2024-09-13T00:18:56.419 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.419 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.893+0000 7ffb5bb40900 0 /usr/bin/ceph-mon: set fsid to 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:18:56.419 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: RocksDB version: 6.15.5 2024-09-13T00:18:56.419 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.419 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T00:18:56.419 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Compile date Sep 12 2024 2024-09-13T00:18:56.419 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: DB SUMMARY 2024-09-13T00:18:56.419 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.419 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: DB Session ID: ACS7H1U9I31WI2JC8ILS 2024-09-13T00:18:56.419 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.419 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2024-09-13T00:18:56.419 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.419 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2024-09-13T00:18:56.420 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.420 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.error_if_exists: 0 2024-09-13T00:18:56.420 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.create_if_missing: 1 2024-09-13T00:18:56.420 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.paranoid_checks: 1 2024-09-13T00:18:56.420 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T00:18:56.420 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.env: 0x55bdb1676440 2024-09-13T00:18:56.420 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.fs: Posix File System 2024-09-13T00:18:56.420 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.info_log: 0x55bdb2192660 2024-09-13T00:18:56.420 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-13T00:18:56.420 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.statistics: (nil) 2024-09-13T00:18:56.420 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.use_fsync: 0 2024-09-13T00:18:56.420 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_log_file_size: 0 2024-09-13T00:18:56.421 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T00:18:56.421 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T00:18:56.421 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-13T00:18:56.421 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-13T00:18:56.421 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.allow_fallocate: 1 2024-09-13T00:18:56.421 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-13T00:18:56.421 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-13T00:18:56.421 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.use_direct_reads: 0 2024-09-13T00:18:56.421 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T00:18:56.421 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.create_missing_column_families: 0 2024-09-13T00:18:56.421 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.db_log_dir: 2024-09-13T00:18:56.421 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-13T00:18:56.422 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T00:18:56.422 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T00:18:56.422 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T00:18:56.422 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T00:18:56.422 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T00:18:56.422 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T00:18:56.422 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.advise_random_on_open: 1 2024-09-13T00:18:56.422 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-13T00:18:56.422 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.write_buffer_manager: 0x55bdb21dbe90 2024-09-13T00:18:56.422 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T00:18:56.422 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T00:18:56.422 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T00:18:56.423 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T00:18:56.423 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.rate_limiter: (nil) 2024-09-13T00:18:56.423 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T00:18:56.423 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-13T00:18:56.423 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-13T00:18:56.423 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-13T00:18:56.423 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.unordered_write: 0 2024-09-13T00:18:56.423 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T00:18:56.423 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T00:18:56.423 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T00:18:56.423 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T00:18:56.423 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.row_cache: None 2024-09-13T00:18:56.423 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.wal_filter: None 2024-09-13T00:18:56.424 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T00:18:56.424 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-13T00:18:56.424 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.preserve_deletes: 0 2024-09-13T00:18:56.424 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.two_write_queues: 0 2024-09-13T00:18:56.424 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.manual_wal_flush: 0 2024-09-13T00:18:56.424 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.atomic_flush: 0 2024-09-13T00:18:56.424 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T00:18:56.424 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T00:18:56.424 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T00:18:56.424 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.log_readahead_size: 0 2024-09-13T00:18:56.425 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T00:18:56.425 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-13T00:18:56.425 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T00:18:56.425 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T00:18:56.425 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-13T00:18:56.425 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-13T00:18:56.425 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_background_jobs: 2 2024-09-13T00:18:56.425 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_background_compactions: -1 2024-09-13T00:18:56.425 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_subcompactions: 1 2024-09-13T00:18:56.425 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T00:18:56.425 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T00:18:56.425 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T00:18:56.426 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_total_wal_size: 0 2024-09-13T00:18:56.426 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T00:18:56.426 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T00:18:56.426 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T00:18:56.426 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T00:18:56.426 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_open_files: -1 2024-09-13T00:18:56.426 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.bytes_per_sync: 0 2024-09-13T00:18:56.426 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T00:18:56.426 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T00:18:56.426 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-13T00:18:56.426 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_background_flushes: -1 2024-09-13T00:18:56.426 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Compression algorithms supported: 2024-09-13T00:18:56.426 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: kZSTD supported: 0 2024-09-13T00:18:56.427 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: kXpressCompression supported: 0 2024-09-13T00:18:56.427 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: kBZip2Compression supported: 0 2024-09-13T00:18:56.427 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T00:18:56.427 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: kLZ4Compression supported: 1 2024-09-13T00:18:56.427 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: kZlibCompression supported: 1 2024-09-13T00:18:56.427 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-13T00:18:56.427 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: kSnappyCompression supported: 1 2024-09-13T00:18:56.427 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T00:18:56.427 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: [db/db_impl/db_impl_open.cc:281] Creating manifest 1 2024-09-13T00:18:56.427 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.427 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2024-09-13T00:18:56.427 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.427 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T00:18:56.428 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.428 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T00:18:56.428 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.merge_operator: 2024-09-13T00:18:56.428 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compaction_filter: None 2024-09-13T00:18:56.428 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compaction_filter_factory: None 2024-09-13T00:18:56.428 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-13T00:18:56.428 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T00:18:56.428 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T00:18:56.428 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55bdb2132ee0) 2024-09-13T00:18:56.428 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks: 1 2024-09-13T00:18:56.428 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T00:18:56.428 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T00:18:56.429 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr pin_top_level_index_and_filter: 1 2024-09-13T00:18:56.429 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr index_type: 0 2024-09-13T00:18:56.429 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr data_block_index_type: 0 2024-09-13T00:18:56.429 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr index_shortening: 1 2024-09-13T00:18:56.429 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr data_block_hash_table_util_ratio: 0.750000 2024-09-13T00:18:56.429 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr hash_index_allow_collision: 1 2024-09-13T00:18:56.429 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr checksum: 1 2024-09-13T00:18:56.429 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr no_block_cache: 0 2024-09-13T00:18:56.429 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr block_cache: 0x55bdb216b1f0 2024-09-13T00:18:56.429 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr block_cache_name: BinnedLRUCache 2024-09-13T00:18:56.429 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr block_cache_options: 2024-09-13T00:18:56.429 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr capacity : 536870912 2024-09-13T00:18:56.430 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr num_shard_bits : 4 2024-09-13T00:18:56.430 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr strict_capacity_limit : 0 2024-09-13T00:18:56.430 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr high_pri_pool_ratio: 0.000 2024-09-13T00:18:56.430 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr block_cache_compressed: (nil) 2024-09-13T00:18:56.430 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr persistent_cache: (nil) 2024-09-13T00:18:56.430 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr block_size: 4096 2024-09-13T00:18:56.430 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr block_size_deviation: 10 2024-09-13T00:18:56.430 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr block_restart_interval: 16 2024-09-13T00:18:56.430 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr index_block_restart_interval: 1 2024-09-13T00:18:56.430 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr metadata_block_size: 4096 2024-09-13T00:18:56.430 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr partition_filters: 0 2024-09-13T00:18:56.430 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr use_delta_encoding: 1 2024-09-13T00:18:56.431 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T00:18:56.431 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr whole_key_filtering: 1 2024-09-13T00:18:56.431 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr verify_compression: 0 2024-09-13T00:18:56.431 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr read_amp_bytes_per_bit: 0 2024-09-13T00:18:56.431 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr format_version: 4 2024-09-13T00:18:56.431 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr enable_index_compression: 1 2024-09-13T00:18:56.431 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr block_align: 0 2024-09-13T00:18:56.431 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.431 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-13T00:18:56.431 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-13T00:18:56.431 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compression: NoCompression 2024-09-13T00:18:56.431 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-13T00:18:56.432 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-13T00:18:56.432 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T00:18:56.432 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.num_levels: 7 2024-09-13T00:18:56.432 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T00:18:56.432 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T00:18:56.432 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T00:18:56.432 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T00:18:56.432 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T00:18:56.432 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T00:18:56.432 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T00:18:56.432 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T00:18:56.432 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T00:18:56.433 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T00:18:56.433 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T00:18:56.433 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compression_opts.level: 32767 2024-09-13T00:18:56.433 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-13T00:18:56.433 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T00:18:56.433 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T00:18:56.433 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T00:18:56.433 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compression_opts.enabled: false 2024-09-13T00:18:56.433 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T00:18:56.433 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T00:18:56.433 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T00:18:56.433 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-13T00:18:56.433 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T00:18:56.434 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T00:18:56.434 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T00:18:56.434 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T00:18:56.434 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T00:18:56.434 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T00:18:56.434 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T00:18:56.434 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T00:18:56.434 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T00:18:56.434 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T00:18:56.434 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T00:18:56.434 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T00:18:56.434 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T00:18:56.434 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.arena_block_size: 4194304 2024-09-13T00:18:56.435 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T00:18:56.435 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T00:18:56.435 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T00:18:56.435 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-13T00:18:56.435 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T00:18:56.435 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T00:18:56.435 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T00:18:56.435 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T00:18:56.435 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T00:18:56.435 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T00:18:56.435 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T00:18:56.435 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T00:18:56.435 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T00:18:56.436 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T00:18:56.436 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T00:18:56.436 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.inplace_update_support: 0 2024-09-13T00:18:56.436 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T00:18:56.436 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T00:18:56.436 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T00:18:56.436 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T00:18:56.436 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.bloom_locality: 0 2024-09-13T00:18:56.436 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.max_successive_merges: 0 2024-09-13T00:18:56.436 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T00:18:56.436 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-13T00:18:56.436 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.force_consistency_checks: 1 2024-09-13T00:18:56.437 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-13T00:18:56.437 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.ttl: 2592000 2024-09-13T00:18:56.437 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T00:18:56.437 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.enable_blob_files: false 2024-09-13T00:18:56.437 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.min_blob_size: 0 2024-09-13T00:18:56.437 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.blob_file_size: 268435456 2024-09-13T00:18:56.437 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T00:18:56.437 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T00:18:56.437 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T00:18:56.437 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T00:18:56.437 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.437 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-13T00:18:56.437 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.438 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.897+0000 7ffb5bb40900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 3 2024-09-13T00:18:56.438 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.438 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.901+0000 7ffb5bb40900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55bdb219c700 2024-09-13T00:18:56.438 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.901+0000 7ffb5bb40900 4 rocksdb: DB pointer 0x55bdb216e000 2024-09-13T00:18:56.438 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.901+0000 7ffb532ce640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T00:18:56.438 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.901+0000 7ffb532ce640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T00:18:56.438 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr ** DB Stats ** 2024-09-13T00:18:56.438 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:18:56.438 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T00:18:56.438 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T00:18:56.438 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T00:18:56.438 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T00:18:56.438 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T00:18:56.439 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T00:18:56.439 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.439 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-09-13T00:18:56.439 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T00:18:56.439 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:18:56.439 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T00:18:56.439 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T00:18:56.439 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.439 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-09-13T00:18:56.439 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T00:18:56.439 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:18:56.439 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:18:56.440 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T00:18:56.440 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T00:18:56.440 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2024-09-13T00:18:56.440 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T00:18:56.440 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2024-09-13T00:18:56.440 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:18:56.440 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:18:56.440 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T00:18:56.440 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.440 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2024-09-13T00:18:56.440 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.440 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-09-13T00:18:56.440 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T00:18:56.441 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:18:56.441 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T00:18:56.441 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T00:18:56.441 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.441 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-09-13T00:18:56.441 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T00:18:56.441 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:18:56.441 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:18:56.441 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T00:18:56.441 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T00:18:56.441 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2024-09-13T00:18:56.441 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T00:18:56.442 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2024-09-13T00:18:56.442 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:18:56.442 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:18:56.442 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T00:18:56.442 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.442 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2024-09-13T00:18:56.442 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr 2024-09-13T00:18:56.442 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.901+0000 7ffb5bb40900 4 rocksdb: [db/db_impl/db_impl.cc:446] Shutdown: canceling all background work 2024-09-13T00:18:56.442 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.901+0000 7ffb5bb40900 4 rocksdb: [db/db_impl/db_impl.cc:625] Shutdown complete 2024-09-13T00:18:56.442 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph-mon: stderr debug 2024-09-13T00:18:55.901+0000 7ffb5bb40900 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2024-09-13T00:18:56.442 INFO:teuthology.orchestra.run.smithi077.stdout:create mon.a on 2024-09-13T00:18:56.672 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2024-09-13T00:18:56.881 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2024-09-13T00:18:57.093 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-9218306c-7165-11ef-bceb-c7b262605968.target -> /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968.target. 2024-09-13T00:18:57.093 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-9218306c-7165-11ef-bceb-c7b262605968.target -> /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968.target. 2024-09-13T00:18:57.307 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.a 2024-09-13T00:18:57.308 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stderr Failed to reset failed state of unit ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.a.service: Unit ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.a.service not loaded. 2024-09-13T00:18:57.527 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968.target.wants/ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.a.service -> /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service. 2024-09-13T00:18:57.537 INFO:teuthology.orchestra.run.smithi077.stdout:firewalld does not appear to be present 2024-09-13T00:18:57.537 INFO:teuthology.orchestra.run.smithi077.stdout:Not possible to enable service . firewalld.service is not available 2024-09-13T00:18:57.537 INFO:teuthology.orchestra.run.smithi077.stdout:Waiting for mon to start... 2024-09-13T00:18:57.537 INFO:teuthology.orchestra.run.smithi077.stdout:Waiting for mon... 2024-09-13T00:18:57.700 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:18:57 smithi077 systemd[1]: Started Ceph mon.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:18:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:18:59 smithi077 bash[22113]: cluster 2024-09-13T00:18:59.375989+0000 mon.a (mon.0) 0 : cluster [INF] mkfs 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:18:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:18:59 smithi077 bash[22113]: cluster 2024-09-13T00:18:59.346150+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T00:19:00.119 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout cluster: 2024-09-13T00:19:00.119 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout id: 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:19:00.119 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-09-13T00:19:00.119 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout 2024-09-13T00:19:00.120 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout services: 2024-09-13T00:19:00.120 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.16042s) 2024-09-13T00:19:00.120 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-09-13T00:19:00.120 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-09-13T00:19:00.120 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout 2024-09-13T00:19:00.120 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout data: 2024-09-13T00:19:00.120 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-09-13T00:19:00.121 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-09-13T00:19:00.121 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-09-13T00:19:00.121 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout pgs: 2024-09-13T00:19:00.121 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout 2024-09-13T00:19:00.121 INFO:teuthology.orchestra.run.smithi077.stdout:mon is available 2024-09-13T00:19:00.121 INFO:teuthology.orchestra.run.smithi077.stdout:Assimilating anything we can from ceph.conf... 2024-09-13T00:19:00.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:00 smithi077 bash[22113]: cluster 2024-09-13T00:18:59.374956+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T00:19:00.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:00 smithi077 bash[22113]: cluster 2024-09-13T00:18:59.375193+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:19:00.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:00 smithi077 bash[22113]: cluster 2024-09-13T00:18:59.377543+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap 2024-09-13T00:19:00.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:00 smithi077 bash[22113]: cluster 2024-09-13T00:18:59.379869+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2024-09-13T00:19:00.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:00 smithi077 bash[22113]: cluster 2024-09-13T00:18:59.380287+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2024-09-13T00:19:00.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:00 smithi077 bash[22113]: audit 2024-09-13T00:18:59.535322+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.77:0/4281350612' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-13T00:19:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:01 smithi077 bash[22113]: audit 2024-09-13T00:19:01.312334+0000 mon.a (mon.0) 8 : audit [INF] from='client.? 172.21.15.77:0/2249386671' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T00:19:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:01 smithi077 bash[22113]: audit 2024-09-13T00:19:01.316600+0000 mon.a (mon.0) 9 : audit [INF] from='client.? 172.21.15.77:0/2249386671' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-13T00:19:01.795 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout 2024-09-13T00:19:01.795 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout [global] 2024-09-13T00:19:01.795 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout fsid = 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:19:01.795 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-13T00:19:01.795 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.77:3300,v1:172.21.15.77:6789] 2024-09-13T00:19:01.795 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-13T00:19:01.795 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-13T00:19:01.795 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-13T00:19:01.796 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-13T00:19:01.796 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout 2024-09-13T00:19:01.796 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-13T00:19:01.796 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-13T00:19:01.796 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout 2024-09-13T00:19:01.796 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout [osd] 2024-09-13T00:19:01.796 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-13T00:19:01.796 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-13T00:19:01.796 INFO:teuthology.orchestra.run.smithi077.stdout:Generating new minimal ceph.conf... 2024-09-13T00:19:03.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:02 smithi077 bash[22113]: audit 2024-09-13T00:19:02.887642+0000 mon.a (mon.0) 10 : audit [DBG] from='client.? 172.21.15.77:0/2919662172' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:19:03.391 INFO:teuthology.orchestra.run.smithi077.stdout:Restarting the monitor... 2024-09-13T00:19:03.696 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:03 smithi077 systemd[1]: Stopping Ceph mon.a for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:19:03.696 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:03 smithi077 bash[22113]: debug 2024-09-13T00:19:03.505+0000 7fb1635a2640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T00:19:03.696 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:03 smithi077 bash[22113]: debug 2024-09-13T00:19:03.505+0000 7fb1635a2640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-13T00:19:04.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:03 smithi077 bash[22500]: ceph-9218306c-7165-11ef-bceb-c7b262605968-mon-a 2024-09-13T00:19:04.057 INFO:teuthology.orchestra.run.smithi077.stdout:Setting public_network to 172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32 in mon config section 2024-09-13T00:19:04.386 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:04 smithi077 bash[22561]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-mon-a 2024-09-13T00:19:04.386 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:04 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T00:19:04.386 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:04 smithi077 systemd[1]: Stopped Ceph mon.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:19:04.386 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:04 smithi077 systemd[1]: Started Ceph mon.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:19:05.706 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.449+0000 7f651e655900 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-13T00:19:05.706 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.449+0000 7f651e655900 0 ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 7 2024-09-13T00:19:05.706 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.449+0000 7f651e655900 0 pidfile_write: ignore empty --pid-file 2024-09-13T00:19:05.707 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 0 load: jerasure load: lrc 2024-09-13T00:19:05.707 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: RocksDB version: 6.15.5 2024-09-13T00:19:05.707 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T00:19:05.707 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Compile date Sep 12 2024 2024-09-13T00:19:05.707 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: DB SUMMARY 2024-09-13T00:19:05.707 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: DB Session ID: F9USGHG4SDH2GVQ1YYK9 2024-09-13T00:19:05.707 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: CURRENT file: CURRENT 2024-09-13T00:19:05.707 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: IDENTITY file: IDENTITY 2024-09-13T00:19:05.707 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-09-13T00:19:05.707 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-13T00:19:05.708 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83161 ; 2024-09-13T00:19:05.708 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.error_if_exists: 0 2024-09-13T00:19:05.708 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.create_if_missing: 0 2024-09-13T00:19:05.708 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.paranoid_checks: 1 2024-09-13T00:19:05.708 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T00:19:05.708 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.env: 0x56186b2db440 2024-09-13T00:19:05.708 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.fs: Posix File System 2024-09-13T00:19:05.708 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.info_log: 0x56186d954ee0 2024-09-13T00:19:05.708 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-13T00:19:05.708 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.statistics: (nil) 2024-09-13T00:19:05.709 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.use_fsync: 0 2024-09-13T00:19:05.709 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_log_file_size: 0 2024-09-13T00:19:05.709 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T00:19:05.709 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T00:19:05.711 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-13T00:19:05.711 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-13T00:19:05.711 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.allow_fallocate: 1 2024-09-13T00:19:05.711 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-13T00:19:05.711 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-13T00:19:05.712 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.use_direct_reads: 0 2024-09-13T00:19:05.712 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T00:19:05.712 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.create_missing_column_families: 0 2024-09-13T00:19:05.712 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.db_log_dir: 2024-09-13T00:19:05.712 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-09-13T00:19:05.712 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T00:19:05.712 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T00:19:05.712 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T00:19:05.712 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T00:19:05.712 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T00:19:05.712 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T00:19:05.712 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.advise_random_on_open: 1 2024-09-13T00:19:05.712 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-13T00:19:05.713 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.write_buffer_manager: 0x56186da127e0 2024-09-13T00:19:05.713 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T00:19:05.713 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T00:19:05.713 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T00:19:05.713 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T00:19:05.713 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.rate_limiter: (nil) 2024-09-13T00:19:05.713 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T00:19:05.713 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-13T00:19:05.714 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-13T00:19:05.714 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-13T00:19:05.714 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.unordered_write: 0 2024-09-13T00:19:05.714 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T00:19:05.714 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T00:19:05.714 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T00:19:05.714 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T00:19:05.714 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.row_cache: None 2024-09-13T00:19:05.714 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.wal_filter: None 2024-09-13T00:19:05.714 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T00:19:05.715 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-13T00:19:05.715 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.preserve_deletes: 0 2024-09-13T00:19:05.715 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.two_write_queues: 0 2024-09-13T00:19:05.715 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.manual_wal_flush: 0 2024-09-13T00:19:05.715 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.atomic_flush: 0 2024-09-13T00:19:05.715 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T00:19:05.715 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T00:19:05.715 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T00:19:05.715 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.log_readahead_size: 0 2024-09-13T00:19:05.715 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T00:19:05.715 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-13T00:19:05.715 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T00:19:05.715 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T00:19:05.716 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-13T00:19:05.716 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-13T00:19:05.716 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_background_jobs: 2 2024-09-13T00:19:05.716 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_background_compactions: -1 2024-09-13T00:19:05.716 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_subcompactions: 1 2024-09-13T00:19:05.716 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T00:19:05.716 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T00:19:05.716 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T00:19:05.716 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_total_wal_size: 0 2024-09-13T00:19:05.716 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T00:19:05.716 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T00:19:05.716 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T00:19:05.717 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T00:19:05.717 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_open_files: -1 2024-09-13T00:19:05.717 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.bytes_per_sync: 0 2024-09-13T00:19:05.717 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T00:19:05.717 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T00:19:05.717 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-13T00:19:05.717 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_background_flushes: -1 2024-09-13T00:19:05.717 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Compression algorithms supported: 2024-09-13T00:19:05.717 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: kZSTD supported: 0 2024-09-13T00:19:05.718 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: kXpressCompression supported: 0 2024-09-13T00:19:05.718 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: kBZip2Compression supported: 0 2024-09-13T00:19:05.718 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T00:19:05.718 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: kLZ4Compression supported: 1 2024-09-13T00:19:05.718 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: kZlibCompression supported: 1 2024-09-13T00:19:05.718 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-13T00:19:05.718 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: kSnappyCompression supported: 1 2024-09-13T00:19:05.718 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T00:19:05.718 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-09-13T00:19:05.719 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T00:19:05.719 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T00:19:05.719 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.merge_operator: 2024-09-13T00:19:05.721 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compaction_filter: None 2024-09-13T00:19:05.721 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compaction_filter_factory: None 2024-09-13T00:19:05.721 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-13T00:19:05.722 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T00:19:05.722 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T00:19:05.722 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56186d8fda38) 2024-09-13T00:19:05.722 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: cache_index_and_filter_blocks: 1 2024-09-13T00:19:05.722 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T00:19:05.722 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T00:19:05.722 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: pin_top_level_index_and_filter: 1 2024-09-13T00:19:05.722 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: index_type: 0 2024-09-13T00:19:05.722 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: data_block_index_type: 0 2024-09-13T00:19:05.722 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: index_shortening: 1 2024-09-13T00:19:05.723 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: data_block_hash_table_util_ratio: 0.750000 2024-09-13T00:19:05.723 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: hash_index_allow_collision: 1 2024-09-13T00:19:05.723 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: checksum: 1 2024-09-13T00:19:05.723 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: no_block_cache: 0 2024-09-13T00:19:05.723 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: block_cache: 0x56186d9311f0 2024-09-13T00:19:05.723 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: block_cache_name: BinnedLRUCache 2024-09-13T00:19:05.723 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: block_cache_options: 2024-09-13T00:19:05.723 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: capacity : 536870912 2024-09-13T00:19:05.723 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: num_shard_bits : 4 2024-09-13T00:19:05.723 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: strict_capacity_limit : 0 2024-09-13T00:19:05.723 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: high_pri_pool_ratio: 0.000 2024-09-13T00:19:05.723 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: block_cache_compressed: (nil) 2024-09-13T00:19:05.724 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: persistent_cache: (nil) 2024-09-13T00:19:05.724 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: block_size: 4096 2024-09-13T00:19:05.724 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: block_size_deviation: 10 2024-09-13T00:19:05.724 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: block_restart_interval: 16 2024-09-13T00:19:05.724 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: index_block_restart_interval: 1 2024-09-13T00:19:05.724 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: metadata_block_size: 4096 2024-09-13T00:19:05.724 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: partition_filters: 0 2024-09-13T00:19:05.724 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: use_delta_encoding: 1 2024-09-13T00:19:05.724 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T00:19:05.724 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: whole_key_filtering: 1 2024-09-13T00:19:05.724 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: verify_compression: 0 2024-09-13T00:19:05.725 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: read_amp_bytes_per_bit: 0 2024-09-13T00:19:05.725 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: format_version: 4 2024-09-13T00:19:05.725 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: enable_index_compression: 1 2024-09-13T00:19:05.725 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: block_align: 0 2024-09-13T00:19:05.725 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-13T00:19:05.725 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-13T00:19:05.725 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compression: NoCompression 2024-09-13T00:19:05.725 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-13T00:19:05.725 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-13T00:19:05.725 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T00:19:05.725 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.num_levels: 7 2024-09-13T00:19:05.725 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T00:19:05.725 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T00:19:05.726 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T00:19:05.726 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T00:19:05.726 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T00:19:05.726 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T00:19:05.726 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T00:19:05.726 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T00:19:05.726 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T00:19:05.726 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T00:19:05.726 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T00:19:05.726 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compression_opts.level: 32767 2024-09-13T00:19:05.726 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-13T00:19:05.727 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T00:19:05.727 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T00:19:05.727 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T00:19:05.727 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compression_opts.enabled: false 2024-09-13T00:19:05.727 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T00:19:05.727 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T00:19:05.727 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T00:19:05.727 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-13T00:19:05.727 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T00:19:05.727 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T00:19:05.727 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T00:19:05.728 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T00:19:05.728 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T00:19:05.728 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T00:19:05.728 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T00:19:05.728 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T00:19:05.728 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T00:19:05.728 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T00:19:05.728 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T00:19:05.728 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T00:19:05.728 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T00:19:05.728 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.arena_block_size: 4194304 2024-09-13T00:19:05.728 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T00:19:05.729 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T00:19:05.729 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T00:19:05.729 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-13T00:19:05.729 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T00:19:05.729 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T00:19:05.729 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T00:19:05.729 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T00:19:05.729 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T00:19:05.729 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T00:19:05.729 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T00:19:05.729 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T00:19:05.729 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T00:19:05.729 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T00:19:05.730 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T00:19:05.730 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.inplace_update_support: 0 2024-09-13T00:19:05.730 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T00:19:05.730 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T00:19:05.730 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T00:19:05.730 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T00:19:05.730 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.bloom_locality: 0 2024-09-13T00:19:05.730 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.max_successive_merges: 0 2024-09-13T00:19:05.730 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T00:19:05.730 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-13T00:19:05.730 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.force_consistency_checks: 1 2024-09-13T00:19:05.730 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-13T00:19:05.731 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.ttl: 2592000 2024-09-13T00:19:05.731 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T00:19:05.731 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.enable_blob_files: false 2024-09-13T00:19:05.731 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.min_blob_size: 0 2024-09-13T00:19:05.731 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.blob_file_size: 268435456 2024-09-13T00:19:05.731 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T00:19:05.731 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T00:19:05.731 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T00:19:05.731 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T00:19:05.731 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-09-13T00:19:05.731 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-09-13T00:19:05.731 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726186745457773, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-09-13T00:19:05.731 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 4 rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-09-13T00:19:05.732 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.453+0000 7f651e655900 3 rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T00:19:05.732 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.457+0000 7f651e655900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726186745459056, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79752, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77991, "index_size": 223, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9999, "raw_average_key_size": 47, "raw_value_size": 72311, "raw_average_value_size": 342, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726186745, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "40b38235-3fb2-48fb-91d5-ab45c7566701", "db_session_id": "F9USGHG4SDH2GVQ1YYK9"}} 2024-09-13T00:19:05.732 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.457+0000 7f651e655900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-09-13T00:19:05.732 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.457+0000 7f651e655900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726186745459641, "job": 1, "event": "recovery_finished"} 2024-09-13T00:19:05.732 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.457+0000 7f651e655900 4 rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T00:19:05.732 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.457+0000 7f651e655900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x56186d966700 2024-09-13T00:19:05.732 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.457+0000 7f651e655900 4 rocksdb: DB pointer 0x56186d934000 2024-09-13T00:19:05.732 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.457+0000 7f6514421640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T00:19:05.732 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.457+0000 7f6514421640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T00:19:05.732 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: ** DB Stats ** 2024-09-13T00:19:05.732 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:19:05.732 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T00:19:05.732 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T00:19:05.733 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T00:19:05.733 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T00:19:05.733 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T00:19:05.733 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T00:19:05.733 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: ** Compaction Stats [default] ** 2024-09-13T00:19:05.733 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T00:19:05.733 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:19:05.733 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: L0 2/0 79.67 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 89.3 0.00 0.00 1 0.001 0 0 2024-09-13T00:19:05.733 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Sum 2/0 79.67 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 89.3 0.00 0.00 1 0.001 0 0 2024-09-13T00:19:05.733 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 89.3 0.00 0.00 1 0.001 0 0 2024-09-13T00:19:05.733 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: ** Compaction Stats [default] ** 2024-09-13T00:19:05.733 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T00:19:05.734 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:19:05.734 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 89.3 0.00 0.00 1 0.001 0 0 2024-09-13T00:19:05.734 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:19:05.734 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T00:19:05.734 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T00:19:05.734 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T00:19:05.734 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T00:19:05.734 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T00:19:05.734 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Cumulative compaction: 0.00 GB write, 14.06 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:19:05.734 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Interval compaction: 0.00 GB write, 14.06 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:19:05.734 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T00:19:05.734 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T00:19:05.734 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: ** Compaction Stats [default] ** 2024-09-13T00:19:05.735 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T00:19:05.735 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:19:05.735 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: L0 2/0 79.67 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 89.3 0.00 0.00 1 0.001 0 0 2024-09-13T00:19:05.735 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Sum 2/0 79.67 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 89.3 0.00 0.00 1 0.001 0 0 2024-09-13T00:19:05.735 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T00:19:05.735 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: ** Compaction Stats [default] ** 2024-09-13T00:19:05.735 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T00:19:05.735 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:19:05.735 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 89.3 0.00 0.00 1 0.001 0 0 2024-09-13T00:19:05.735 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:19:05.735 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T00:19:05.735 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T00:19:05.735 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T00:19:05.736 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T00:19:05.736 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T00:19:05.736 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Cumulative compaction: 0.00 GB write, 13.90 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:19:05.736 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:19:05.736 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T00:19:05.736 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T00:19:05.736 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.457+0000 7f651e655900 0 starting mon.a rank 0 at public addrs [v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0] at bind addrs [v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:19:05.736 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.461+0000 7f651e655900 1 mon.a@-1(???) e1 preinit fsid 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:19:05.736 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.461+0000 7f651e655900 0 mon.a@-1(???).mds e1 new map 2024-09-13T00:19:05.736 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.461+0000 7f651e655900 0 mon.a@-1(???).mds e1 print_map 2024-09-13T00:19:05.736 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: e1 2024-09-13T00:19:05.736 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T00:19:05.736 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T00:19:05.736 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: legacy client fscid: -1 2024-09-13T00:19:05.737 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: 2024-09-13T00:19:05.737 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: No filesystems configured 2024-09-13T00:19:05.737 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.461+0000 7f651e655900 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T00:19:05.737 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.461+0000 7f651e655900 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T00:19:05.737 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.461+0000 7f651e655900 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T00:19:05.737 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.461+0000 7f651e655900 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T00:19:05.737 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: debug 2024-09-13T00:19:05.461+0000 7f651e655900 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-13T00:19:05.737 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: cluster 2024-09-13T00:19:05.486999+0000 mon.a (mon.0) 1 : cluster [INF] mon.a is new leader, mons a in quorum (ranks 0) 2024-09-13T00:19:05.737 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: cluster 2024-09-13T00:19:05.487061+0000 mon.a (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:19:05.737 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: cluster 2024-09-13T00:19:05.487467+0000 mon.a (mon.0) 3 : cluster [DBG] fsmap 2024-09-13T00:19:05.737 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: cluster 2024-09-13T00:19:05.487493+0000 mon.a (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2024-09-13T00:19:05.737 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:05 smithi077 bash[22596]: cluster 2024-09-13T00:19:05.487963+0000 mon.a (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2024-09-13T00:19:06.289 INFO:teuthology.orchestra.run.smithi077.stdout:Wrote config to /etc/ceph/ceph.conf 2024-09-13T00:19:06.289 INFO:teuthology.orchestra.run.smithi077.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-13T00:19:06.289 INFO:teuthology.orchestra.run.smithi077.stdout:Creating mgr... 2024-09-13T00:19:06.290 INFO:teuthology.orchestra.run.smithi077.stdout:Verifying port 0.0.0.0:9283 ... 2024-09-13T00:19:06.290 INFO:teuthology.orchestra.run.smithi077.stdout:Verifying port 0.0.0.0:8765 ... 2024-09-13T00:19:06.550 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:06 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:19:06.550 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:06 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:19:06.550 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:06 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:19:06.557 INFO:teuthology.orchestra.run.smithi077.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-9218306c-7165-11ef-bceb-c7b262605968@mgr.y 2024-09-13T00:19:06.557 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stderr Failed to reset failed state of unit ceph-9218306c-7165-11ef-bceb-c7b262605968@mgr.y.service: Unit ceph-9218306c-7165-11ef-bceb-c7b262605968@mgr.y.service not loaded. 2024-09-13T00:19:06.739 INFO:teuthology.orchestra.run.smithi077.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968.target.wants/ceph-9218306c-7165-11ef-bceb-c7b262605968@mgr.y.service -> /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service. 2024-09-13T00:19:06.746 INFO:teuthology.orchestra.run.smithi077.stdout:firewalld does not appear to be present 2024-09-13T00:19:06.746 INFO:teuthology.orchestra.run.smithi077.stdout:Not possible to enable service . firewalld.service is not available 2024-09-13T00:19:06.746 INFO:teuthology.orchestra.run.smithi077.stdout:firewalld does not appear to be present 2024-09-13T00:19:06.746 INFO:teuthology.orchestra.run.smithi077.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-09-13T00:19:06.747 INFO:teuthology.orchestra.run.smithi077.stdout:Waiting for mgr to start... 2024-09-13T00:19:06.747 INFO:teuthology.orchestra.run.smithi077.stdout:Waiting for mgr... 2024-09-13T00:19:06.862 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:06 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:19:06.862 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:06 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:19:06.862 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:06 smithi077 systemd[1]: Started Ceph mgr.y for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:19:06.862 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:06 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:19:06.862 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:06 smithi077 bash[22596]: audit 2024-09-13T00:19:05.697044+0000 mon.a (mon.0) 6 : audit [INF] from='client.? 172.21.15.77:0/3195343913' entity='client.admin' 2024-09-13T00:19:08.388 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:08 smithi077 bash[22845]: debug 2024-09-13T00:19:08.353+0000 7f43a52edec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T00:19:08.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:08 smithi077 bash[22596]: audit 2024-09-13T00:19:08.379609+0000 mon.a (mon.0) 7 : audit [DBG] from='client.? 172.21.15.77:0/897763038' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T00:19:08.957 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout 2024-09-13T00:19:08.957 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout { 2024-09-13T00:19:08.957 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "fsid": "9218306c-7165-11ef-bceb-c7b262605968", 2024-09-13T00:19:08.957 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T00:19:08.957 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T00:19:08.957 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T00:19:08.957 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T00:19:08.957 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:08.957 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T00:19:08.958 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T00:19:08.958 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout 0 2024-09-13T00:19:08.958 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout ], 2024-09-13T00:19:08.958 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T00:19:08.958 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T00:19:08.958 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout ], 2024-09-13T00:19:08.958 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-09-13T00:19:08.958 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T00:19:08.958 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T00:19:08.958 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-13T00:19:08.959 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T00:19:08.959 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:08.959 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T00:19:08.959 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T00:19:08.959 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T00:19:08.959 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T00:19:08.959 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T00:19:08.959 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T00:19:08.959 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T00:19:08.959 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T00:19:08.960 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:08.960 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T00:19:08.960 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T00:19:08.960 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T00:19:08.960 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T00:19:08.960 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T00:19:08.960 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T00:19:08.960 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T00:19:08.960 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T00:19:08.960 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T00:19:08.960 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:08.960 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T00:19:08.961 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T00:19:08.961 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T00:19:08.961 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T00:19:08.961 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:08.961 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T00:19:08.961 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-13T00:19:08.961 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T00:19:08.961 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T00:19:08.961 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T00:19:08.962 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T00:19:08.962 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T00:19:08.962 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout ], 2024-09-13T00:19:08.962 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T00:19:08.962 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:08.963 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T00:19:08.963 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T00:19:08.963 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T00:18:59.376413+0000", 2024-09-13T00:19:08.963 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T00:19:08.963 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:08.963 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T00:19:08.963 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout } 2024-09-13T00:19:08.964 INFO:teuthology.orchestra.run.smithi077.stdout:mgr not available, waiting (1/15)... 2024-09-13T00:19:09.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:09 smithi077 bash[22845]: debug 2024-09-13T00:19:09.525+0000 7f43a52edec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T00:19:09.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:09 smithi077 bash[22845]: debug 2024-09-13T00:19:09.585+0000 7f43a52edec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T00:19:09.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:09 smithi077 bash[22845]: debug 2024-09-13T00:19:09.733+0000 7f43a52edec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T00:19:10.638 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:10 smithi077 bash[22845]: debug 2024-09-13T00:19:10.349+0000 7f43a52edec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T00:19:10.638 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:10 smithi077 bash[22845]: debug 2024-09-13T00:19:10.417+0000 7f43a52edec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T00:19:10.638 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:10 smithi077 bash[22845]: debug 2024-09-13T00:19:10.485+0000 7f43a52edec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T00:19:11.002 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:10 smithi077 bash[22845]: debug 2024-09-13T00:19:10.637+0000 7f43a52edec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T00:19:11.002 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:10 smithi077 bash[22845]: debug 2024-09-13T00:19:10.709+0000 7f43a52edec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T00:19:11.002 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:10 smithi077 bash[22845]: debug 2024-09-13T00:19:10.777+0000 7f43a52edec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T00:19:11.002 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:11 smithi077 bash[22845]: debug 2024-09-13T00:19:11.001+0000 7f43a52edec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T00:19:11.279 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:11 smithi077 bash[22845]: debug 2024-09-13T00:19:11.069+0000 7f43a52edec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T00:19:11.279 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:11 smithi077 bash[22845]: debug 2024-09-13T00:19:11.233+0000 7f43a52edec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T00:19:11.697 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:11 smithi077 bash[22845]: debug 2024-09-13T00:19:11.305+0000 7f43a52edec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T00:19:11.697 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:11 smithi077 bash[22845]: debug 2024-09-13T00:19:11.537+0000 7f43a52edec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:19:12.029 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:11 smithi077 bash[22845]: debug 2024-09-13T00:19:11.697+0000 7f43a52edec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T00:19:12.029 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:11 smithi077 bash[22845]: debug 2024-09-13T00:19:11.773+0000 7f43a52edec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T00:19:12.029 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:11 smithi077 bash[22845]: debug 2024-09-13T00:19:11.929+0000 7f43a52edec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:19:12.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:12 smithi077 bash[22596]: audit 2024-09-13T00:19:12.111498+0000 mon.a (mon.0) 8 : audit [DBG] from='client.? 172.21.15.77:0/2567458433' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T00:19:12.595 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout 2024-09-13T00:19:12.596 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout { 2024-09-13T00:19:12.596 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "fsid": "9218306c-7165-11ef-bceb-c7b262605968", 2024-09-13T00:19:12.596 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T00:19:12.596 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T00:19:12.596 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T00:19:12.596 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T00:19:12.596 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:12.596 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T00:19:12.596 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T00:19:12.596 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout 0 2024-09-13T00:19:12.596 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout ], 2024-09-13T00:19:12.596 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T00:19:12.597 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T00:19:12.597 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout ], 2024-09-13T00:19:12.597 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2024-09-13T00:19:12.597 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T00:19:12.597 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T00:19:12.597 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-13T00:19:12.597 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T00:19:12.597 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:12.597 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T00:19:12.597 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T00:19:12.597 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T00:19:12.597 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T00:19:12.597 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T00:19:12.598 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T00:19:12.598 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T00:19:12.598 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T00:19:12.598 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:12.598 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T00:19:12.598 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T00:19:12.598 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T00:19:12.598 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T00:19:12.598 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T00:19:12.598 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T00:19:12.598 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T00:19:12.598 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T00:19:12.598 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T00:19:12.598 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:12.599 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T00:19:12.599 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T00:19:12.599 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T00:19:12.599 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T00:19:12.599 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:12.599 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T00:19:12.599 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-13T00:19:12.599 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T00:19:12.599 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T00:19:12.599 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T00:19:12.599 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T00:19:12.599 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T00:19:12.599 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout ], 2024-09-13T00:19:12.599 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T00:19:12.600 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:12.600 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T00:19:12.600 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T00:19:12.600 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T00:18:59.376413+0000", 2024-09-13T00:19:12.600 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T00:19:12.600 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:12.600 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T00:19:12.600 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout } 2024-09-13T00:19:12.600 INFO:teuthology.orchestra.run.smithi077.stdout:mgr not available, waiting (2/15)... 2024-09-13T00:19:12.961 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:12 smithi077 bash[22845]: debug 2024-09-13T00:19:12.693+0000 7f43a52edec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T00:19:12.961 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:12 smithi077 bash[22845]: debug 2024-09-13T00:19:12.765+0000 7f43a52edec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T00:19:13.279 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:12 smithi077 bash[22845]: debug 2024-09-13T00:19:12.961+0000 7f43a52edec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T00:19:13.279 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:13 smithi077 bash[22845]: debug 2024-09-13T00:19:13.029+0000 7f43a52edec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T00:19:13.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:13 smithi077 bash[22845]: debug 2024-09-13T00:19:13.337+0000 7f43a52edec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T00:19:13.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:13 smithi077 bash[22845]: debug 2024-09-13T00:19:13.413+0000 7f43a52edec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T00:19:13.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:13 smithi077 bash[22596]: cluster 2024-09-13T00:19:13.436620+0000 mon.a (mon.0) 9 : cluster [INF] Activating manager daemon y 2024-09-13T00:19:13.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:13 smithi077 bash[22596]: cluster 2024-09-13T00:19:13.443102+0000 mon.a (mon.0) 10 : cluster [DBG] mgrmap e2: y(active, starting, since 0.00668336s) 2024-09-13T00:19:13.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:13 smithi077 bash[22596]: audit 2024-09-13T00:19:13.444292+0000 mon.a (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.77:0/4269985290' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T00:19:13.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:13 smithi077 bash[22596]: audit 2024-09-13T00:19:13.444467+0000 mon.a (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.77:0/4269985290' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T00:19:13.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:13 smithi077 bash[22596]: audit 2024-09-13T00:19:13.444559+0000 mon.a (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.77:0/4269985290' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T00:19:13.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:13 smithi077 bash[22596]: audit 2024-09-13T00:19:13.444649+0000 mon.a (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.77:0/4269985290' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:19:13.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:13 smithi077 bash[22596]: audit 2024-09-13T00:19:13.446006+0000 mon.a (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.77:0/4269985290' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T00:19:13.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:13 smithi077 bash[22596]: cluster 2024-09-13T00:19:13.473016+0000 mon.a (mon.0) 16 : cluster [INF] Manager daemon y is now available 2024-09-13T00:19:13.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:13 smithi077 bash[22596]: audit 2024-09-13T00:19:13.486536+0000 mon.a (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.77:0/4269985290' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:19:14.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:14 smithi077 bash[22596]: audit 2024-09-13T00:19:13.490646+0000 mon.a (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.77:0/4269985290' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:19:14.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:14 smithi077 bash[22596]: audit 2024-09-13T00:19:13.492320+0000 mon.a (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.77:0/4269985290' entity='mgr.y' 2024-09-13T00:19:14.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:14 smithi077 bash[22596]: audit 2024-09-13T00:19:13.494756+0000 mon.a (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.77:0/4269985290' entity='mgr.y' 2024-09-13T00:19:14.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:14 smithi077 bash[22596]: audit 2024-09-13T00:19:13.496805+0000 mon.a (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.77:0/4269985290' entity='mgr.y' 2024-09-13T00:19:14.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:14 smithi077 bash[22596]: cluster 2024-09-13T00:19:14.451916+0000 mon.a (mon.0) 22 : cluster [DBG] mgrmap e3: y(active, since 1.01549s) 2024-09-13T00:19:16.525 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout 2024-09-13T00:19:16.525 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout { 2024-09-13T00:19:16.525 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "fsid": "9218306c-7165-11ef-bceb-c7b262605968", 2024-09-13T00:19:16.526 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "health": { 2024-09-13T00:19:16.526 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-13T00:19:16.526 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-13T00:19:16.526 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-13T00:19:16.526 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:16.526 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-13T00:19:16.526 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-13T00:19:16.526 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout 0 2024-09-13T00:19:16.526 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout ], 2024-09-13T00:19:16.526 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-13T00:19:16.526 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "a" 2024-09-13T00:19:16.526 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout ], 2024-09-13T00:19:16.527 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "quorum_age": 10, 2024-09-13T00:19:16.527 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-13T00:19:16.527 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T00:19:16.527 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-09-13T00:19:16.527 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-13T00:19:16.527 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:16.527 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-13T00:19:16.527 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T00:19:16.527 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-13T00:19:16.527 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-13T00:19:16.527 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-13T00:19:16.527 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-13T00:19:16.528 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-13T00:19:16.528 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-13T00:19:16.531 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:16.531 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-13T00:19:16.531 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-13T00:19:16.531 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-13T00:19:16.532 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-13T00:19:16.532 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-13T00:19:16.532 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-13T00:19:16.532 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-13T00:19:16.532 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-13T00:19:16.532 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-13T00:19:16.532 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:16.532 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-13T00:19:16.533 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T00:19:16.533 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-13T00:19:16.533 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-13T00:19:16.533 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:16.533 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-13T00:19:16.533 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T00:19:16.533 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-13T00:19:16.533 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-13T00:19:16.533 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-13T00:19:16.533 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-13T00:19:16.533 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "restful" 2024-09-13T00:19:16.533 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout ], 2024-09-13T00:19:16.534 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T00:19:16.534 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:16.534 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-13T00:19:16.534 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-13T00:19:16.534 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "modified": "2024-09-13T00:18:59.376413+0000", 2024-09-13T00:19:16.534 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-13T00:19:16.534 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout }, 2024-09-13T00:19:16.534 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-13T00:19:16.534 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout } 2024-09-13T00:19:16.534 INFO:teuthology.orchestra.run.smithi077.stdout:mgr is available 2024-09-13T00:19:16.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:16 smithi077 bash[22596]: cluster 2024-09-13T00:19:15.496189+0000 mon.a (mon.0) 23 : cluster [DBG] mgrmap e4: y(active, since 2s) 2024-09-13T00:19:16.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:16 smithi077 bash[22596]: audit 2024-09-13T00:19:16.072047+0000 mon.a (mon.0) 24 : audit [DBG] from='client.? 172.21.15.77:0/104757661' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-13T00:19:18.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:17 smithi077 bash[22596]: audit 2024-09-13T00:19:17.912515+0000 mon.a (mon.0) 25 : audit [INF] from='client.? 172.21.15.77:0/2993237376' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-13T00:19:18.398 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout 2024-09-13T00:19:18.398 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout [global] 2024-09-13T00:19:18.398 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout fsid = 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:19:18.399 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout mon_cluster_log_file_level = debug 2024-09-13T00:19:18.399 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-13T00:19:18.399 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-13T00:19:18.399 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-13T00:19:18.399 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-13T00:19:18.399 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout 2024-09-13T00:19:18.400 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-13T00:19:18.400 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-13T00:19:18.400 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout 2024-09-13T00:19:18.400 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout [osd] 2024-09-13T00:19:18.400 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-13T00:19:18.400 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-13T00:19:18.400 INFO:teuthology.orchestra.run.smithi077.stdout:Enabling cephadm module... 2024-09-13T00:19:20.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:19 smithi077 bash[22596]: audit 2024-09-13T00:19:19.775803+0000 mon.a (mon.0) 26 : audit [INF] from='client.? 172.21.15.77:0/2386311462' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-13T00:19:20.279 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:19 smithi077 bash[22845]: ignoring --setuser ceph since I am not root 2024-09-13T00:19:20.279 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:19 smithi077 bash[22845]: ignoring --setgroup ceph since I am not root 2024-09-13T00:19:20.279 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:19 smithi077 bash[22845]: debug 2024-09-13T00:19:19.969+0000 7fb1d5966ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T00:19:21.137 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:20 smithi077 bash[22596]: audit 2024-09-13T00:19:19.831235+0000 mon.a (mon.0) 27 : audit [INF] from='client.? 172.21.15.77:0/2386311462' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-13T00:19:21.137 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:20 smithi077 bash[22596]: cluster 2024-09-13T00:19:19.831290+0000 mon.a (mon.0) 28 : cluster [DBG] mgrmap e5: y(active, since 6s) 2024-09-13T00:19:21.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:21 smithi077 bash[22845]: debug 2024-09-13T00:19:21.137+0000 7fb1d5966ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T00:19:21.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:21 smithi077 bash[22845]: debug 2024-09-13T00:19:21.201+0000 7fb1d5966ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T00:19:21.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:21 smithi077 bash[22845]: debug 2024-09-13T00:19:21.357+0000 7fb1d5966ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T00:19:21.986 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:21 smithi077 bash[22596]: audit 2024-09-13T00:19:21.721856+0000 mon.a (mon.0) 29 : audit [DBG] from='client.? 172.21.15.77:0/3639249530' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T00:19:22.274 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:21 smithi077 bash[22845]: debug 2024-09-13T00:19:21.985+0000 7fb1d5966ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T00:19:22.275 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:22 smithi077 bash[22845]: debug 2024-09-13T00:19:22.053+0000 7fb1d5966ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T00:19:22.275 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:22 smithi077 bash[22845]: debug 2024-09-13T00:19:22.117+0000 7fb1d5966ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T00:19:22.280 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout { 2024-09-13T00:19:22.280 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-09-13T00:19:22.281 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T00:19:22.281 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-13T00:19:22.281 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-13T00:19:22.281 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout } 2024-09-13T00:19:22.281 INFO:teuthology.orchestra.run.smithi077.stdout:Waiting for the mgr to restart... 2024-09-13T00:19:22.281 INFO:teuthology.orchestra.run.smithi077.stdout:Waiting for mgr epoch 5... 2024-09-13T00:19:22.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:22 smithi077 bash[22845]: debug 2024-09-13T00:19:22.272+0000 7fb1d5966ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T00:19:22.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:22 smithi077 bash[22845]: debug 2024-09-13T00:19:22.348+0000 7fb1d5966ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T00:19:22.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:22 smithi077 bash[22845]: debug 2024-09-13T00:19:22.416+0000 7fb1d5966ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T00:19:22.946 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:22 smithi077 bash[22845]: debug 2024-09-13T00:19:22.640+0000 7fb1d5966ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T00:19:22.946 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:22 smithi077 bash[22845]: debug 2024-09-13T00:19:22.704+0000 7fb1d5966ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T00:19:22.946 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:22 smithi077 bash[22845]: debug 2024-09-13T00:19:22.868+0000 7fb1d5966ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T00:19:23.204 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:22 smithi077 bash[22845]: debug 2024-09-13T00:19:22.944+0000 7fb1d5966ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T00:19:23.204 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:23 smithi077 bash[22845]: debug 2024-09-13T00:19:23.176+0000 7fb1d5966ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:19:23.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:23 smithi077 bash[22845]: debug 2024-09-13T00:19:23.340+0000 7fb1d5966ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T00:19:23.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:23 smithi077 bash[22845]: debug 2024-09-13T00:19:23.416+0000 7fb1d5966ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T00:19:24.029 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:23 smithi077 bash[22845]: debug 2024-09-13T00:19:23.568+0000 7fb1d5966ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:19:24.586 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:24 smithi077 bash[22845]: debug 2024-09-13T00:19:24.316+0000 7fb1d5966ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T00:19:24.586 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:24 smithi077 bash[22845]: debug 2024-09-13T00:19:24.392+0000 7fb1d5966ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T00:19:24.968 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:24 smithi077 bash[22845]: debug 2024-09-13T00:19:24.584+0000 7fb1d5966ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T00:19:24.968 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:24 smithi077 bash[22845]: debug 2024-09-13T00:19:24.656+0000 7fb1d5966ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T00:19:25.279 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:24 smithi077 bash[22845]: debug 2024-09-13T00:19:24.964+0000 7fb1d5966ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T00:19:25.279 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:25 smithi077 bash[22845]: debug 2024-09-13T00:19:25.040+0000 7fb1d5966ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T00:19:25.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:25 smithi077 bash[22596]: cluster 2024-09-13T00:19:25.064917+0000 mon.a (mon.0) 30 : cluster [INF] Active manager daemon y restarted 2024-09-13T00:19:25.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:25 smithi077 bash[22596]: cluster 2024-09-13T00:19:25.065338+0000 mon.a (mon.0) 31 : cluster [INF] Activating manager daemon y 2024-09-13T00:19:25.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:25 smithi077 bash[22596]: cluster 2024-09-13T00:19:25.072242+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2024-09-13T00:19:25.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:25 smithi077 bash[22596]: cluster 2024-09-13T00:19:25.072523+0000 mon.a (mon.0) 33 : cluster [DBG] mgrmap e6: y(active, starting, since 0.00735336s) 2024-09-13T00:19:25.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:25 smithi077 bash[22596]: audit 2024-09-13T00:19:25.074871+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:19:25.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:25 smithi077 bash[22596]: audit 2024-09-13T00:19:25.076240+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T00:19:25.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:25 smithi077 bash[22596]: audit 2024-09-13T00:19:25.077301+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T00:19:25.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:25 smithi077 bash[22596]: audit 2024-09-13T00:19:25.077609+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T00:19:25.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:25 smithi077 bash[22596]: audit 2024-09-13T00:19:25.077799+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T00:19:25.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:25 smithi077 bash[22596]: cluster 2024-09-13T00:19:25.104355+0000 mon.a (mon.0) 39 : cluster [INF] Manager daemon y is now available 2024-09-13T00:19:26.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22596]: cephadm 2024-09-13T00:19:25.115487+0000 mgr.y (mgr.14118) 1 : cephadm [INF] Found migration_current of "None". Setting to last migration. 2024-09-13T00:19:26.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22596]: audit 2024-09-13T00:19:25.120751+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' 2024-09-13T00:19:26.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22596]: audit 2024-09-13T00:19:25.131948+0000 mon.a (mon.0) 41 : audit [INF] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' 2024-09-13T00:19:26.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22596]: audit 2024-09-13T00:19:25.134596+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:19:26.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22596]: audit 2024-09-13T00:19:25.138392+0000 mon.a (mon.0) 43 : audit [DBG] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:19:26.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22596]: audit 2024-09-13T00:19:25.147350+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:19:26.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22596]: audit 2024-09-13T00:19:25.149615+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:19:26.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22596]: cluster 2024-09-13T00:19:26.077390+0000 mon.a (mon.0) 46 : cluster [DBG] mgrmap e7: y(active, since 1.01222s) 2024-09-13T00:19:26.628 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout { 2024-09-13T00:19:26.629 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-09-13T00:19:26.629 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-13T00:19:26.629 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout } 2024-09-13T00:19:26.629 INFO:teuthology.orchestra.run.smithi077.stdout:mgr epoch 5 is available 2024-09-13T00:19:26.629 INFO:teuthology.orchestra.run.smithi077.stdout:Setting orchestrator backend to cephadm... 2024-09-13T00:19:27.030 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: debug 2024-09-13T00:19:26.800+0000 7fb187d36640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:00:19:26] ENGINE Error in HTTPServer.serve 2024-09-13T00:19:27.030 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: Traceback (most recent call last): 2024-09-13T00:19:27.030 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:19:27.030 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: self._connections.run(self.expiration_interval) 2024-09-13T00:19:27.030 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:19:27.030 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: self._run(expiration_interval) 2024-09-13T00:19:27.030 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:19:27.030 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:19:27.030 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:19:27.030 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:19:27.030 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:19:27.031 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: s = self.context.wrap_socket( 2024-09-13T00:19:27.031 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:19:27.031 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: return self.sslsocket_class._create( 2024-09-13T00:19:27.031 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:19:27.031 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: self.do_handshake() 2024-09-13T00:19:27.031 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:19:27.031 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: self._sslobj.do_handshake() 2024-09-13T00:19:27.031 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:26 smithi077 bash[22845]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:19:28.078 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: audit 2024-09-13T00:19:26.075620+0000 mgr.y (mgr.14118) 2 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-13T00:19:28.078 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: audit 2024-09-13T00:19:26.086236+0000 mgr.y (mgr.14118) 3 : audit [DBG] from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-13T00:19:28.078 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: cephadm 2024-09-13T00:19:26.682941+0000 mgr.y (mgr.14118) 4 : cephadm [INF] [13/Sep/2024:00:19:26] ENGINE Bus STARTING 2024-09-13T00:19:28.078 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: cephadm 2024-09-13T00:19:26.800423+0000 mgr.y (mgr.14118) 5 : cephadm [INF] [13/Sep/2024:00:19:26] ENGINE Serving on https://172.21.15.77:7150 2024-09-13T00:19:28.078 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: cephadm 2024-09-13T00:19:26.800522+0000 mgr.y (mgr.14118) 6 : cephadm [INF] [13/Sep/2024:00:19:26] ENGINE Bus STARTED 2024-09-13T00:19:28.078 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: cephadm 2024-09-13T00:19:26.802767+0000 mgr.y (mgr.14118) 7 : cephadm [ERR] [13/Sep/2024:00:19:26] ENGINE Error in HTTPServer.serve 2024-09-13T00:19:28.078 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: Traceback (most recent call last): 2024-09-13T00:19:28.078 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:19:28.078 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: self._connections.run(self.expiration_interval) 2024-09-13T00:19:28.078 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:19:28.079 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: self._run(expiration_interval) 2024-09-13T00:19:28.079 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:19:28.079 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:19:28.079 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:19:28.079 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:19:28.079 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:19:28.079 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: s = self.context.wrap_socket( 2024-09-13T00:19:28.079 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:19:28.079 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: return self.sslsocket_class._create( 2024-09-13T00:19:28.079 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:19:28.079 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: self.do_handshake() 2024-09-13T00:19:28.079 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:19:28.079 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: self._sslobj.do_handshake() 2024-09-13T00:19:28.079 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:19:28.080 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: audit 2024-09-13T00:19:26.806283+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' 2024-09-13T00:19:28.080 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:27 smithi077 bash[22596]: audit 2024-09-13T00:19:26.807366+0000 mon.a (mon.0) 48 : audit [DBG] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:19:29.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:28 smithi077 bash[22596]: cluster 2024-09-13T00:19:27.816398+0000 mon.a (mon.0) 49 : cluster [DBG] mgrmap e8: y(active, since 2s) 2024-09-13T00:19:29.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:28 smithi077 bash[22596]: audit 2024-09-13T00:19:28.059476+0000 mon.a (mon.0) 50 : audit [INF] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' 2024-09-13T00:19:29.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:28 smithi077 bash[22596]: audit 2024-09-13T00:19:28.075781+0000 mon.a (mon.0) 51 : audit [DBG] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:19:30.109 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:29 smithi077 bash[22596]: audit 2024-09-13T00:19:28.049661+0000 mgr.y (mgr.14118) 8 : audit [DBG] from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:19:30.451 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout value unchanged 2024-09-13T00:19:30.451 INFO:teuthology.orchestra.run.smithi077.stdout:Generating ssh key... 2024-09-13T00:19:31.270 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:30 smithi077 bash[22596]: audit 2024-09-13T00:19:29.953274+0000 mgr.y (mgr.14118) 9 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:19:32.280 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: Generating public/private rsa key pair. 2024-09-13T00:19:32.280 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: Your identification has been saved in /tmp/tmp1ji14oum/key 2024-09-13T00:19:32.280 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: Your public key has been saved in /tmp/tmp1ji14oum/key.pub 2024-09-13T00:19:32.280 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: The key fingerprint is: 2024-09-13T00:19:32.280 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: SHA256:CvNGVcValhQJnTUFne92FPeHilIhrRryzM8AzdzftU4 ceph-9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:19:32.280 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: The key's randomart image is: 2024-09-13T00:19:32.280 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: +---[RSA 3072]----+ 2024-09-13T00:19:32.281 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: | .o*+*++o| 2024-09-13T00:19:32.281 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: | ..o O +o| 2024-09-13T00:19:32.281 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: | + ..o = .=| 2024-09-13T00:19:32.281 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: | o =.o o o =| 2024-09-13T00:19:32.281 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: | o*.oSo o o +.| 2024-09-13T00:19:32.281 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: | =*.. o o E +| 2024-09-13T00:19:32.281 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: | ++ . o ..| 2024-09-13T00:19:32.281 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: | . o . | 2024-09-13T00:19:32.281 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: | | 2024-09-13T00:19:32.281 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:31 smithi077 bash[22845]: +----[SHA256]-----+ 2024-09-13T00:19:33.186 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:32 smithi077 bash[22596]: audit 2024-09-13T00:19:31.772076+0000 mgr.y (mgr.14118) 10 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:19:33.186 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:32 smithi077 bash[22596]: cephadm 2024-09-13T00:19:31.772450+0000 mgr.y (mgr.14118) 11 : cephadm [INF] Generating ssh key... 2024-09-13T00:19:33.186 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:32 smithi077 bash[22596]: audit 2024-09-13T00:19:31.837858+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' 2024-09-13T00:19:33.186 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:32 smithi077 bash[22596]: audit 2024-09-13T00:19:31.842947+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' 2024-09-13T00:19:34.233 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDXqlBKwBffj0YCRXYaA5sy9fkc/6skDiWtNWnBgZR/fkMVWHd/TVjlGzN0jW0dM5Rfw6j8vfHz6nVRzlzsi9QTYJjRrZlimJoW56PhhLEvRFRqUqdPSsom02FlXfEdgDMNM12q1hi3dcHHSeHsDhKyDn1Jh8ob+cjxh0gH3dq4O+qUTgzwp74jfpDX0gzji/eF/AEVNvEXCtmSQD5MhOFzyUYlLr8pTcwD5tMFZObJ6mPKGrSAtBkUUZhPXcH5I7PfFjI3o0lwhuWQKGYbK43b4v9+qjrtU+LhM5fZMRM6PXchz2SeCVWgAxsAzZpJPUymsZN+0JlP/fzCk68aVBLFE/LWWaR/zzl6xXQY9Cmj0iBfJ1n7r9T3QR5sryNfUUklCtMtRD/NJPlO/5jqxg0cCEu0LV6L3v9y6826ZzKpcMwDjGHFvlH8dbCmFqeJvxLFaKoVv0QfFNK1qrfFuelJDPeQl+4hHItd8MuGooYCMIqMjkYOv73lxSXqWehpmXk= ceph-9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:19:34.233 INFO:teuthology.orchestra.run.smithi077.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-13T00:19:34.233 INFO:teuthology.orchestra.run.smithi077.stdout:Adding key to root@localhost authorized_keys... 2024-09-13T00:19:34.234 INFO:teuthology.orchestra.run.smithi077.stdout:Adding host smithi077... 2024-09-13T00:19:34.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:33 smithi077 bash[22596]: audit 2024-09-13T00:19:33.658860+0000 mgr.y (mgr.14118) 12 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:19:36.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:35 smithi077 bash[22596]: audit 2024-09-13T00:19:35.550753+0000 mgr.y (mgr.14118) 13 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi077", "addr": "172.21.15.77", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:19:37.161 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:36 smithi077 bash[22596]: cephadm 2024-09-13T00:19:36.521078+0000 mgr.y (mgr.14118) 14 : cephadm [INF] Deploying cephadm binary to smithi077 2024-09-13T00:19:39.457 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout Added host 'smithi077' with addr '172.21.15.77' 2024-09-13T00:19:39.457 INFO:teuthology.orchestra.run.smithi077.stdout:Deploying unmanaged mon service... 2024-09-13T00:19:40.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:39 smithi077 bash[22596]: audit 2024-09-13T00:19:38.669313+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' 2024-09-13T00:19:40.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:39 smithi077 bash[22596]: cephadm 2024-09-13T00:19:38.669667+0000 mgr.y (mgr.14118) 15 : cephadm [INF] Added host smithi077 2024-09-13T00:19:40.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:39 smithi077 bash[22596]: audit 2024-09-13T00:19:38.670002+0000 mon.a (mon.0) 55 : audit [DBG] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:19:42.130 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-09-13T00:19:42.130 INFO:teuthology.orchestra.run.smithi077.stdout:Deploying unmanaged mgr service... 2024-09-13T00:19:43.018 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:42 smithi077 bash[22596]: audit 2024-09-13T00:19:41.600754+0000 mgr.y (mgr.14118) 16 : audit [DBG] from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:19:43.018 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:42 smithi077 bash[22596]: cephadm 2024-09-13T00:19:41.603985+0000 mgr.y (mgr.14118) 17 : cephadm [INF] Saving service mon spec with placement count:5 2024-09-13T00:19:43.018 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:42 smithi077 bash[22596]: audit 2024-09-13T00:19:41.606961+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' 2024-09-13T00:19:43.018 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:42 smithi077 bash[22596]: audit 2024-09-13T00:19:41.612733+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' 2024-09-13T00:19:43.018 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:42 smithi077 bash[22596]: audit 2024-09-13T00:19:41.855480+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' 2024-09-13T00:19:43.997 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-09-13T00:19:44.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:44 smithi077 bash[22596]: audit 2024-09-13T00:19:43.494197+0000 mgr.y (mgr.14118) 18 : audit [DBG] from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:19:44.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:44 smithi077 bash[22596]: cephadm 2024-09-13T00:19:43.496285+0000 mgr.y (mgr.14118) 19 : cephadm [INF] Saving service mgr spec with placement count:2 2024-09-13T00:19:44.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:44 smithi077 bash[22596]: audit 2024-09-13T00:19:43.499602+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' 2024-09-13T00:19:46.672 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:46 smithi077 bash[22596]: cluster 2024-09-13T00:19:45.078806+0000 mgr.y (mgr.14118) 20 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:19:46.672 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:46 smithi077 bash[22596]: audit 2024-09-13T00:19:45.334675+0000 mon.a (mon.0) 60 : audit [INF] from='client.? 172.21.15.77:0/3746177505' entity='client.admin' 2024-09-13T00:19:47.702 INFO:teuthology.orchestra.run.smithi077.stdout:Enabling the dashboard module... 2024-09-13T00:19:48.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:47 smithi077 bash[22596]: audit 2024-09-13T00:19:46.768140+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' 2024-09-13T00:19:48.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:47 smithi077 bash[22596]: audit 2024-09-13T00:19:47.162507+0000 mon.a (mon.0) 62 : audit [INF] from='client.? 172.21.15.77:0/1229464428' entity='client.admin' 2024-09-13T00:19:48.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:47 smithi077 bash[22596]: audit 2024-09-13T00:19:47.359866+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.14118 172.21.15.77:0/3232304818' entity='mgr.y' 2024-09-13T00:19:49.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:48 smithi077 bash[22596]: cluster 2024-09-13T00:19:47.079073+0000 mgr.y (mgr.14118) 21 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:19:50.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:49 smithi077 bash[22596]: audit 2024-09-13T00:19:49.204204+0000 mon.a (mon.0) 64 : audit [INF] from='client.? 172.21.15.77:0/169672461' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-13T00:19:50.030 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:49 smithi077 bash[22845]: ignoring --setuser ceph since I am not root 2024-09-13T00:19:50.030 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:49 smithi077 bash[22845]: ignoring --setgroup ceph since I am not root 2024-09-13T00:19:50.030 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:49 smithi077 bash[22845]: debug 2024-09-13T00:19:49.931+0000 7fa1e7338ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T00:19:51.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:50 smithi077 bash[22596]: audit 2024-09-13T00:19:49.782347+0000 mon.a (mon.0) 65 : audit [INF] from='client.? 172.21.15.77:0/169672461' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-13T00:19:51.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:50 smithi077 bash[22596]: cluster 2024-09-13T00:19:49.782419+0000 mon.a (mon.0) 66 : cluster [DBG] mgrmap e9: y(active, since 24s) 2024-09-13T00:19:51.530 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:51 smithi077 bash[22845]: debug 2024-09-13T00:19:51.151+0000 7fa1e7338ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T00:19:51.530 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:51 smithi077 bash[22845]: debug 2024-09-13T00:19:51.215+0000 7fa1e7338ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T00:19:51.530 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:51 smithi077 bash[22845]: debug 2024-09-13T00:19:51.371+0000 7fa1e7338ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T00:19:52.080 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:51 smithi077 bash[22596]: audit 2024-09-13T00:19:51.799173+0000 mon.a (mon.0) 67 : audit [DBG] from='client.? 172.21.15.77:0/1064023808' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-13T00:19:52.080 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:52 smithi077 bash[22845]: debug 2024-09-13T00:19:52.007+0000 7fa1e7338ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T00:19:52.376 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout { 2024-09-13T00:19:52.376 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-09-13T00:19:52.376 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-13T00:19:52.376 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-13T00:19:52.376 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-13T00:19:52.376 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout } 2024-09-13T00:19:52.376 INFO:teuthology.orchestra.run.smithi077.stdout:Waiting for the mgr to restart... 2024-09-13T00:19:52.376 INFO:teuthology.orchestra.run.smithi077.stdout:Waiting for mgr epoch 9... 2024-09-13T00:19:52.377 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:52 smithi077 bash[22845]: debug 2024-09-13T00:19:52.075+0000 7fa1e7338ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T00:19:52.377 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:52 smithi077 bash[22845]: debug 2024-09-13T00:19:52.143+0000 7fa1e7338ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T00:19:52.377 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:52 smithi077 bash[22845]: debug 2024-09-13T00:19:52.295+0000 7fa1e7338ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T00:19:52.378 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:52 smithi077 bash[22845]: debug 2024-09-13T00:19:52.375+0000 7fa1e7338ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T00:19:52.671 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:52 smithi077 bash[22845]: debug 2024-09-13T00:19:52.443+0000 7fa1e7338ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T00:19:52.978 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:52 smithi077 bash[22845]: debug 2024-09-13T00:19:52.667+0000 7fa1e7338ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T00:19:52.978 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:52 smithi077 bash[22845]: debug 2024-09-13T00:19:52.731+0000 7fa1e7338ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T00:19:52.978 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:52 smithi077 bash[22845]: debug 2024-09-13T00:19:52.899+0000 7fa1e7338ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T00:19:52.978 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:52 smithi077 bash[22845]: debug 2024-09-13T00:19:52.975+0000 7fa1e7338ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T00:19:53.280 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:53 smithi077 bash[22845]: debug 2024-09-13T00:19:53.207+0000 7fa1e7338ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:19:53.608 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:53 smithi077 bash[22845]: debug 2024-09-13T00:19:53.367+0000 7fa1e7338ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T00:19:53.608 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:53 smithi077 bash[22845]: debug 2024-09-13T00:19:53.451+0000 7fa1e7338ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T00:19:54.280 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:53 smithi077 bash[22845]: debug 2024-09-13T00:19:53.603+0000 7fa1e7338ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:19:54.649 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:54 smithi077 bash[22845]: debug 2024-09-13T00:19:54.371+0000 7fa1e7338ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T00:19:54.649 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:54 smithi077 bash[22845]: debug 2024-09-13T00:19:54.447+0000 7fa1e7338ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T00:19:54.650 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:54 smithi077 bash[22845]: debug 2024-09-13T00:19:54.643+0000 7fa1e7338ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T00:19:54.904 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:54 smithi077 bash[22845]: debug 2024-09-13T00:19:54.719+0000 7fa1e7338ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T00:19:55.196 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:55 smithi077 bash[22845]: debug 2024-09-13T00:19:55.039+0000 7fa1e7338ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T00:19:55.197 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:55 smithi077 bash[22845]: debug 2024-09-13T00:19:55.119+0000 7fa1e7338ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T00:19:55.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:55 smithi077 bash[22596]: cluster 2024-09-13T00:19:55.144714+0000 mon.a (mon.0) 68 : cluster [INF] Active manager daemon y restarted 2024-09-13T00:19:55.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:55 smithi077 bash[22596]: cluster 2024-09-13T00:19:55.145438+0000 mon.a (mon.0) 69 : cluster [INF] Activating manager daemon y 2024-09-13T00:19:55.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:55 smithi077 bash[22596]: cluster 2024-09-13T00:19:55.152346+0000 mon.a (mon.0) 70 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2024-09-13T00:19:55.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:55 smithi077 bash[22596]: cluster 2024-09-13T00:19:55.152580+0000 mon.a (mon.0) 71 : cluster [DBG] mgrmap e10: y(active, starting, since 0.00748206s) 2024-09-13T00:19:55.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:55 smithi077 bash[22596]: audit 2024-09-13T00:19:55.155128+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:19:55.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:55 smithi077 bash[22596]: audit 2024-09-13T00:19:55.156425+0000 mon.a (mon.0) 73 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T00:19:55.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:55 smithi077 bash[22596]: audit 2024-09-13T00:19:55.157707+0000 mon.a (mon.0) 74 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T00:19:55.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:55 smithi077 bash[22596]: audit 2024-09-13T00:19:55.157908+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T00:19:55.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:55 smithi077 bash[22596]: audit 2024-09-13T00:19:55.158067+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T00:19:55.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:55 smithi077 bash[22596]: cluster 2024-09-13T00:19:55.184425+0000 mon.a (mon.0) 77 : cluster [INF] Manager daemon y is now available 2024-09-13T00:19:56.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22596]: audit 2024-09-13T00:19:55.201461+0000 mon.a (mon.0) 78 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:19:56.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22596]: audit 2024-09-13T00:19:55.217882+0000 mon.a (mon.0) 79 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:19:56.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22596]: audit 2024-09-13T00:19:55.220578+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:19:56.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22596]: cluster 2024-09-13T00:19:56.158011+0000 mon.a (mon.0) 81 : cluster [DBG] mgrmap e11: y(active, since 1.01291s) 2024-09-13T00:19:56.531 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: debug 2024-09-13T00:19:56.335+0000 7fa197ec5640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:00:19:56] ENGINE Error in HTTPServer.serve 2024-09-13T00:19:56.531 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: Traceback (most recent call last): 2024-09-13T00:19:56.531 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:19:56.531 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: self._connections.run(self.expiration_interval) 2024-09-13T00:19:56.531 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:19:56.531 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: self._run(expiration_interval) 2024-09-13T00:19:56.531 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:19:56.531 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:19:56.531 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:19:56.532 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:19:56.532 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:19:56.532 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: s = self.context.wrap_socket( 2024-09-13T00:19:56.532 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:19:56.532 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: return self.sslsocket_class._create( 2024-09-13T00:19:56.532 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:19:56.532 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: self.do_handshake() 2024-09-13T00:19:56.532 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:19:56.532 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: self._sslobj.do_handshake() 2024-09-13T00:19:56.532 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:19:56 smithi077 bash[22845]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:19:56.768 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout { 2024-09-13T00:19:56.768 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-09-13T00:19:56.768 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-13T00:19:56.768 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout } 2024-09-13T00:19:56.768 INFO:teuthology.orchestra.run.smithi077.stdout:mgr epoch 9 is available 2024-09-13T00:19:56.769 INFO:teuthology.orchestra.run.smithi077.stdout:Generating a dashboard self-signed certificate... 2024-09-13T00:19:57.604 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: cephadm 2024-09-13T00:19:56.220266+0000 mgr.y (mgr.14150) 3 : cephadm [INF] [13/Sep/2024:00:19:56] ENGINE Bus STARTING 2024-09-13T00:19:57.605 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: cephadm 2024-09-13T00:19:56.337950+0000 mgr.y (mgr.14150) 4 : cephadm [INF] [13/Sep/2024:00:19:56] ENGINE Serving on https://172.21.15.77:7150 2024-09-13T00:19:57.605 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: cephadm 2024-09-13T00:19:56.338059+0000 mgr.y (mgr.14150) 5 : cephadm [INF] [13/Sep/2024:00:19:56] ENGINE Bus STARTED 2024-09-13T00:19:57.605 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: cephadm 2024-09-13T00:19:56.339879+0000 mgr.y (mgr.14150) 6 : cephadm [ERR] [13/Sep/2024:00:19:56] ENGINE Error in HTTPServer.serve 2024-09-13T00:19:57.605 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: Traceback (most recent call last): 2024-09-13T00:19:57.605 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:19:57.605 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: self._connections.run(self.expiration_interval) 2024-09-13T00:19:57.605 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:19:57.605 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: self._run(expiration_interval) 2024-09-13T00:19:57.605 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:19:57.605 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:19:57.605 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:19:57.605 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:19:57.606 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:19:57.606 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: s = self.context.wrap_socket( 2024-09-13T00:19:57.606 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:19:57.606 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: return self.sslsocket_class._create( 2024-09-13T00:19:57.606 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:19:57.606 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: self.do_handshake() 2024-09-13T00:19:57.606 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:19:57.606 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: self._sslobj.do_handshake() 2024-09-13T00:19:57.606 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:19:57.606 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:57 smithi077 bash[22596]: audit 2024-09-13T00:19:56.345303+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:19:58.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:58 smithi077 bash[22596]: cluster 2024-09-13T00:19:57.347607+0000 mon.a (mon.0) 83 : cluster [DBG] mgrmap e12: y(active, since 2s) 2024-09-13T00:19:58.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:58 smithi077 bash[22596]: audit 2024-09-13T00:19:58.096068+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:19:58.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:58 smithi077 bash[22596]: audit 2024-09-13T00:19:58.103089+0000 mon.a (mon.0) 85 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:19:58.641 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-09-13T00:19:58.641 INFO:teuthology.orchestra.run.smithi077.stdout:Creating initial admin user... 2024-09-13T00:19:59.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:19:59 smithi077 bash[22596]: audit 2024-09-13T00:19:58.063865+0000 mgr.y (mgr.14150) 7 : audit [DBG] from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:20:00.841 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$JLKzplUl15zSPs0LfKNJLumuXQu5FNTWpNhVEsOH35odhbgnJgfQe", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726186800, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-13T00:20:00.841 INFO:teuthology.orchestra.run.smithi077.stdout:Fetching dashboard port number... 2024-09-13T00:20:01.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:01 smithi077 bash[22596]: audit 2024-09-13T00:19:59.996931+0000 mgr.y (mgr.14150) 8 : audit [DBG] from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:20:01.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:01 smithi077 bash[22596]: audit 2024-09-13T00:20:00.267938+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:02.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:02 smithi077 bash[22596]: cluster 2024-09-13T00:20:01.278644+0000 mon.a (mon.0) 87 : cluster [DBG] mgrmap e13: y(active, since 6s) 2024-09-13T00:20:02.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:02 smithi077 bash[22596]: audit 2024-09-13T00:20:01.831246+0000 mon.a (mon.0) 88 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:02.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:02 smithi077 bash[22596]: audit 2024-09-13T00:20:02.085981+0000 mon.a (mon.0) 89 : audit [DBG] from='client.? 172.21.15.77:0/1754627837' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-13T00:20:02.639 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stdout 8443 2024-09-13T00:20:02.639 INFO:teuthology.orchestra.run.smithi077.stdout:firewalld does not appear to be present 2024-09-13T00:20:02.639 INFO:teuthology.orchestra.run.smithi077.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-13T00:20:02.642 INFO:teuthology.orchestra.run.smithi077.stdout:Ceph Dashboard is now available at: 2024-09-13T00:20:02.642 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:20:02.642 INFO:teuthology.orchestra.run.smithi077.stdout: URL: https://smithi077.front.sepia.ceph.com:8443/ 2024-09-13T00:20:02.642 INFO:teuthology.orchestra.run.smithi077.stdout: User: admin 2024-09-13T00:20:02.642 INFO:teuthology.orchestra.run.smithi077.stdout: Password: 3qkz3vq54o 2024-09-13T00:20:02.642 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:20:02.643 INFO:teuthology.orchestra.run.smithi077.stdout:Saving cluster configuration to /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config directory 2024-09-13T00:20:02.643 INFO:teuthology.orchestra.run.smithi077.stdout:Enabling autotune for osd_memory_target 2024-09-13T00:20:03.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:03 smithi077 bash[22596]: audit 2024-09-13T00:20:02.360072+0000 mon.a (mon.0) 90 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:06.463 INFO:teuthology.orchestra.run.smithi077.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-09-13T00:20:06.464 INFO:teuthology.orchestra.run.smithi077.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-09-13T00:20:06.464 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:20:06.464 INFO:teuthology.orchestra.run.smithi077.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-13T00:20:06.464 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:20:06.464 INFO:teuthology.orchestra.run.smithi077.stdout:Or, if you are only running a single cluster on this host: 2024-09-13T00:20:06.464 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:20:06.464 INFO:teuthology.orchestra.run.smithi077.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-09-13T00:20:06.464 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:20:06.464 INFO:teuthology.orchestra.run.smithi077.stdout:Please consider enabling telemetry to help improve Ceph: 2024-09-13T00:20:06.464 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:20:06.464 INFO:teuthology.orchestra.run.smithi077.stdout: ceph telemetry on 2024-09-13T00:20:06.464 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:20:06.464 INFO:teuthology.orchestra.run.smithi077.stdout:For more information see: 2024-09-13T00:20:06.465 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:20:06.465 INFO:teuthology.orchestra.run.smithi077.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-09-13T00:20:06.465 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:20:06.465 INFO:teuthology.orchestra.run.smithi077.stdout:Bootstrap complete. 2024-09-13T00:20:06.519 INFO:tasks.cephadm:Fetching config... 2024-09-13T00:20:06.520 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:20:06.520 DEBUG:teuthology.orchestra.run.smithi077:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-13T00:20:06.525 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-13T00:20:06.526 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:20:06.526 DEBUG:teuthology.orchestra.run.smithi077:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-13T00:20:06.577 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-13T00:20:06.577 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:20:06.578 DEBUG:teuthology.orchestra.run.smithi077:> sudo dd if=/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/keyring of=/dev/stdout 2024-09-13T00:20:06.631 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-13T00:20:06.632 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:20:06.632 DEBUG:teuthology.orchestra.run.smithi077:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-13T00:20:06.677 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-13T00:20:06.677 DEBUG:teuthology.orchestra.run.smithi077:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDXqlBKwBffj0YCRXYaA5sy9fkc/6skDiWtNWnBgZR/fkMVWHd/TVjlGzN0jW0dM5Rfw6j8vfHz6nVRzlzsi9QTYJjRrZlimJoW56PhhLEvRFRqUqdPSsom02FlXfEdgDMNM12q1hi3dcHHSeHsDhKyDn1Jh8ob+cjxh0gH3dq4O+qUTgzwp74jfpDX0gzji/eF/AEVNvEXCtmSQD5MhOFzyUYlLr8pTcwD5tMFZObJ6mPKGrSAtBkUUZhPXcH5I7PfFjI3o0lwhuWQKGYbK43b4v9+qjrtU+LhM5fZMRM6PXchz2SeCVWgAxsAzZpJPUymsZN+0JlP/fzCk68aVBLFE/LWWaR/zzl6xXQY9Cmj0iBfJ1n7r9T3QR5sryNfUUklCtMtRD/NJPlO/5jqxg0cCEu0LV6L3v9y6826ZzKpcMwDjGHFvlH8dbCmFqeJvxLFaKoVv0QfFNK1qrfFuelJDPeQl+4hHItd8MuGooYCMIqMjkYOv73lxSXqWehpmXk= ceph-9218306c-7165-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T00:20:06.742 INFO:teuthology.orchestra.run.smithi077.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDXqlBKwBffj0YCRXYaA5sy9fkc/6skDiWtNWnBgZR/fkMVWHd/TVjlGzN0jW0dM5Rfw6j8vfHz6nVRzlzsi9QTYJjRrZlimJoW56PhhLEvRFRqUqdPSsom02FlXfEdgDMNM12q1hi3dcHHSeHsDhKyDn1Jh8ob+cjxh0gH3dq4O+qUTgzwp74jfpDX0gzji/eF/AEVNvEXCtmSQD5MhOFzyUYlLr8pTcwD5tMFZObJ6mPKGrSAtBkUUZhPXcH5I7PfFjI3o0lwhuWQKGYbK43b4v9+qjrtU+LhM5fZMRM6PXchz2SeCVWgAxsAzZpJPUymsZN+0JlP/fzCk68aVBLFE/LWWaR/zzl6xXQY9Cmj0iBfJ1n7r9T3QR5sryNfUUklCtMtRD/NJPlO/5jqxg0cCEu0LV6L3v9y6826ZzKpcMwDjGHFvlH8dbCmFqeJvxLFaKoVv0QfFNK1qrfFuelJDPeQl+4hHItd8MuGooYCMIqMjkYOv73lxSXqWehpmXk= ceph-9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:20:06.750 DEBUG:teuthology.orchestra.run.smithi088:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDXqlBKwBffj0YCRXYaA5sy9fkc/6skDiWtNWnBgZR/fkMVWHd/TVjlGzN0jW0dM5Rfw6j8vfHz6nVRzlzsi9QTYJjRrZlimJoW56PhhLEvRFRqUqdPSsom02FlXfEdgDMNM12q1hi3dcHHSeHsDhKyDn1Jh8ob+cjxh0gH3dq4O+qUTgzwp74jfpDX0gzji/eF/AEVNvEXCtmSQD5MhOFzyUYlLr8pTcwD5tMFZObJ6mPKGrSAtBkUUZhPXcH5I7PfFjI3o0lwhuWQKGYbK43b4v9+qjrtU+LhM5fZMRM6PXchz2SeCVWgAxsAzZpJPUymsZN+0JlP/fzCk68aVBLFE/LWWaR/zzl6xXQY9Cmj0iBfJ1n7r9T3QR5sryNfUUklCtMtRD/NJPlO/5jqxg0cCEu0LV6L3v9y6826ZzKpcMwDjGHFvlH8dbCmFqeJvxLFaKoVv0QfFNK1qrfFuelJDPeQl+4hHItd8MuGooYCMIqMjkYOv73lxSXqWehpmXk= ceph-9218306c-7165-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-13T00:20:06.773 INFO:teuthology.orchestra.run.smithi088.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDXqlBKwBffj0YCRXYaA5sy9fkc/6skDiWtNWnBgZR/fkMVWHd/TVjlGzN0jW0dM5Rfw6j8vfHz6nVRzlzsi9QTYJjRrZlimJoW56PhhLEvRFRqUqdPSsom02FlXfEdgDMNM12q1hi3dcHHSeHsDhKyDn1Jh8ob+cjxh0gH3dq4O+qUTgzwp74jfpDX0gzji/eF/AEVNvEXCtmSQD5MhOFzyUYlLr8pTcwD5tMFZObJ6mPKGrSAtBkUUZhPXcH5I7PfFjI3o0lwhuWQKGYbK43b4v9+qjrtU+LhM5fZMRM6PXchz2SeCVWgAxsAzZpJPUymsZN+0JlP/fzCk68aVBLFE/LWWaR/zzl6xXQY9Cmj0iBfJ1n7r9T3QR5sryNfUUklCtMtRD/NJPlO/5jqxg0cCEu0LV6L3v9y6826ZzKpcMwDjGHFvlH8dbCmFqeJvxLFaKoVv0QfFNK1qrfFuelJDPeQl+4hHItd8MuGooYCMIqMjkYOv73lxSXqWehpmXk= ceph-9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:20:06.782 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-13T00:20:07.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:06 smithi077 bash[22596]: audit 2024-09-13T00:20:05.977148+0000 mon.a (mon.0) 91 : audit [INF] from='client.? 172.21.15.77:0/752825612' entity='client.admin' 2024-09-13T00:20:12.656 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:20:13.506 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:13 smithi077 bash[22596]: audit 2024-09-13T00:20:12.111828+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:13.506 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:13 smithi077 bash[22596]: audit 2024-09-13T00:20:12.117614+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:13.506 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:13 smithi077 bash[22596]: audit 2024-09-13T00:20:12.119123+0000 mon.a (mon.0) 94 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi077", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:20:13.506 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:13 smithi077 bash[22596]: audit 2024-09-13T00:20:12.126532+0000 mon.a (mon.0) 95 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:13.507 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:13 smithi077 bash[22596]: audit 2024-09-13T00:20:12.153930+0000 mon.a (mon.0) 96 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:20:13.507 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:13 smithi077 bash[22596]: audit 2024-09-13T00:20:12.161938+0000 mon.a (mon.0) 97 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:15.925 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-13T00:20:15.926 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-13T00:20:16.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:16 smithi077 bash[22596]: cluster 2024-09-13T00:20:15.159316+0000 mgr.y (mgr.14150) 9 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:16.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:16 smithi077 bash[22596]: audit 2024-09-13T00:20:15.335638+0000 mon.a (mon.0) 98 : audit [INF] from='client.? 172.21.15.77:0/2993231493' entity='client.admin' 2024-09-13T00:20:16.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:16 smithi077 bash[22596]: audit 2024-09-13T00:20:15.351467+0000 mon.a (mon.0) 99 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:20:16.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:16 smithi077 bash[22596]: audit 2024-09-13T00:20:15.357778+0000 mon.a (mon.0) 100 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:18.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:18 smithi077 bash[22596]: cluster 2024-09-13T00:20:17.159698+0000 mgr.y (mgr.14150) 10 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:20.637 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:20:20.744 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:20 smithi077 bash[22596]: cluster 2024-09-13T00:20:19.160149+0000 mgr.y (mgr.14150) 11 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:22.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:22 smithi077 bash[22596]: cluster 2024-09-13T00:20:21.160561+0000 mgr.y (mgr.14150) 12 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:24.053 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi088 2024-09-13T00:20:24.053 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-13T00:20:24.053 DEBUG:teuthology.orchestra.run.smithi088:> dd of=/etc/ceph/ceph.conf 2024-09-13T00:20:24.059 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-13T00:20:24.060 DEBUG:teuthology.orchestra.run.smithi088:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:20:24.105 INFO:tasks.cephadm:Adding host smithi088 to orchestrator... 2024-09-13T00:20:24.105 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch host add smithi088 2024-09-13T00:20:24.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:24 smithi077 bash[22596]: cluster 2024-09-13T00:20:23.160859+0000 mgr.y (mgr.14150) 13 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:24.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:24 smithi077 bash[22596]: audit 2024-09-13T00:20:23.434050+0000 mgr.y (mgr.14150) 14 : audit [DBG] from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:20:24.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:24 smithi077 bash[22596]: audit 2024-09-13T00:20:23.437560+0000 mon.a (mon.0) 101 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:24.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:24 smithi077 bash[22596]: audit 2024-09-13T00:20:23.438289+0000 mon.a (mon.0) 102 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:20:24.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:24 smithi077 bash[22596]: audit 2024-09-13T00:20:23.440762+0000 mon.a (mon.0) 103 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:20:24.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:24 smithi077 bash[22596]: audit 2024-09-13T00:20:23.441539+0000 mon.a (mon.0) 104 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:20:24.782 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:24 smithi077 bash[22596]: cephadm 2024-09-13T00:20:23.442686+0000 mgr.y (mgr.14150) 15 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.conf 2024-09-13T00:20:24.782 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:24 smithi077 bash[22596]: cephadm 2024-09-13T00:20:23.528289+0000 mgr.y (mgr.14150) 16 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:20:24.782 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:24 smithi077 bash[22596]: cephadm 2024-09-13T00:20:23.604506+0000 mgr.y (mgr.14150) 17 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:20:24.782 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:24 smithi077 bash[22596]: cephadm 2024-09-13T00:20:23.683229+0000 mgr.y (mgr.14150) 18 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:20:24.782 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:24 smithi077 bash[22596]: audit 2024-09-13T00:20:23.766425+0000 mon.a (mon.0) 105 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:24.782 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:24 smithi077 bash[22596]: audit 2024-09-13T00:20:23.770191+0000 mon.a (mon.0) 106 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:24.782 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:24 smithi077 bash[22596]: audit 2024-09-13T00:20:23.772534+0000 mon.a (mon.0) 107 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:26.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:26 smithi077 bash[22596]: cluster 2024-09-13T00:20:25.161249+0000 mgr.y (mgr.14150) 19 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:28.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:28 smithi077 bash[22596]: cluster 2024-09-13T00:20:27.161665+0000 mgr.y (mgr.14150) 20 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:28.807 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:20:30.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:30 smithi077 bash[22596]: cluster 2024-09-13T00:20:29.162082+0000 mgr.y (mgr.14150) 21 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:32.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:32 smithi077 bash[22596]: cluster 2024-09-13T00:20:31.162512+0000 mgr.y (mgr.14150) 22 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:32.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:32 smithi077 bash[22596]: audit 2024-09-13T00:20:31.840733+0000 mgr.y (mgr.14150) 23 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi088", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:20:33.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:33 smithi077 bash[22596]: cephadm 2024-09-13T00:20:32.826470+0000 mgr.y (mgr.14150) 24 : cephadm [INF] Deploying cephadm binary to smithi088 2024-09-13T00:20:34.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:34 smithi077 bash[22596]: cluster 2024-09-13T00:20:33.162896+0000 mgr.y (mgr.14150) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:35.085 INFO:teuthology.orchestra.run.smithi077.stdout:Added host 'smithi088' with addr '172.21.15.88' 2024-09-13T00:20:35.688 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-13T00:20:36.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:36 smithi077 bash[22596]: audit 2024-09-13T00:20:35.084212+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:36.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:36 smithi077 bash[22596]: cephadm 2024-09-13T00:20:35.085082+0000 mgr.y (mgr.14150) 26 : cephadm [INF] Added host smithi088 2024-09-13T00:20:36.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:36 smithi077 bash[22596]: audit 2024-09-13T00:20:35.085935+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:20:36.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:36 smithi077 bash[22596]: cluster 2024-09-13T00:20:35.163166+0000 mgr.y (mgr.14150) 27 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:36.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:36 smithi077 bash[22596]: audit 2024-09-13T00:20:35.342903+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:38.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:37 smithi077 bash[22596]: audit 2024-09-13T00:20:36.611783+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:38.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:37 smithi077 bash[22596]: audit 2024-09-13T00:20:37.399022+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:39.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:38 smithi077 bash[22596]: cluster 2024-09-13T00:20:37.163481+0000 mgr.y (mgr.14150) 28 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:40.403 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:20:40.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:40 smithi077 bash[22596]: cluster 2024-09-13T00:20:39.163888+0000 mgr.y (mgr.14150) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:43.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:42 smithi077 bash[22596]: cluster 2024-09-13T00:20:41.164285+0000 mgr.y (mgr.14150) 30 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:43.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:42 smithi077 bash[22596]: audit 2024-09-13T00:20:42.590060+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:43.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:42 smithi077 bash[22596]: audit 2024-09-13T00:20:42.597631+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:43.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:42 smithi077 bash[22596]: audit 2024-09-13T00:20:42.604063+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:43.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:42 smithi077 bash[22596]: audit 2024-09-13T00:20:42.609572+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:43.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:42 smithi077 bash[22596]: audit 2024-09-13T00:20:42.610990+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi088", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:20:43.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:42 smithi077 bash[22596]: audit 2024-09-13T00:20:42.613016+0000 mon.a (mon.0) 118 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:20:43.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:42 smithi077 bash[22596]: audit 2024-09-13T00:20:42.614708+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:20:43.358 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:20:43.358 INFO:teuthology.orchestra.run.smithi077.stdout:[{"addr": "172.21.15.77", "hostname": "smithi077", "labels": [], "status": ""}, {"addr": "172.21.15.88", "hostname": "smithi088", "labels": [], "status": ""}] 2024-09-13T00:20:43.936 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:43 smithi077 bash[22596]: cephadm 2024-09-13T00:20:42.617126+0000 mgr.y (mgr.14150) 31 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:20:43.936 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:43 smithi077 bash[22596]: cephadm 2024-09-13T00:20:42.792296+0000 mgr.y (mgr.14150) 32 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:20:43.936 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:43 smithi077 bash[22596]: audit 2024-09-13T00:20:43.183730+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:43.936 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:43 smithi077 bash[22596]: audit 2024-09-13T00:20:43.186808+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:43.936 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:43 smithi077 bash[22596]: audit 2024-09-13T00:20:43.190285+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:43.937 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-13T00:20:43.938 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-13T00:20:45.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:44 smithi077 bash[22596]: cephadm 2024-09-13T00:20:42.939353+0000 mgr.y (mgr.14150) 33 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:20:45.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:44 smithi077 bash[22596]: cephadm 2024-09-13T00:20:43.075392+0000 mgr.y (mgr.14150) 34 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:20:45.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:44 smithi077 bash[22596]: cluster 2024-09-13T00:20:43.164627+0000 mgr.y (mgr.14150) 35 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:45.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:44 smithi077 bash[22596]: audit 2024-09-13T00:20:43.357653+0000 mgr.y (mgr.14150) 36 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:20:47.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:46 smithi077 bash[22596]: cluster 2024-09-13T00:20:45.165090+0000 mgr.y (mgr.14150) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:48.649 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:20:49.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:48 smithi077 bash[22596]: cluster 2024-09-13T00:20:47.165604+0000 mgr.y (mgr.14150) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:51.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:50 smithi077 bash[22596]: cluster 2024-09-13T00:20:49.166054+0000 mgr.y (mgr.14150) 39 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:51.644 INFO:teuthology.orchestra.run.smithi077.stderr:adjusted tunables profile to default 2024-09-13T00:20:52.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:51 smithi077 bash[22596]: audit 2024-09-13T00:20:51.541441+0000 mon.a (mon.0) 123 : audit [INF] from='client.? 172.21.15.77:0/738112169' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T00:20:52.239 INFO:tasks.cephadm:Adding mon.a on smithi077 2024-09-13T00:20:52.239 INFO:tasks.cephadm:Adding mon.c on smithi077 2024-09-13T00:20:52.239 INFO:tasks.cephadm:Adding mon.b on smithi088 2024-09-13T00:20:52.239 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch apply mon '3;smithi077:172.21.15.77=a;smithi077:[v2:172.21.15.77:3301,v1:172.21.15.77:6790]=c;smithi088:172.21.15.88=b' 2024-09-13T00:20:53.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:52 smithi077 bash[22596]: cluster 2024-09-13T00:20:51.166462+0000 mgr.y (mgr.14150) 40 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:53.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:52 smithi077 bash[22596]: audit 2024-09-13T00:20:51.644237+0000 mon.a (mon.0) 124 : audit [INF] from='client.? 172.21.15.77:0/738112169' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T00:20:53.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:52 smithi077 bash[22596]: cluster 2024-09-13T00:20:51.644344+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T00:20:53.452 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:20:55.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:54 smithi077 bash[22596]: cluster 2024-09-13T00:20:53.166873+0000 mgr.y (mgr.14150) 41 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:55.919 INFO:teuthology.orchestra.run.smithi088.stdout:Scheduled mon update... 2024-09-13T00:20:56.605 DEBUG:teuthology.orchestra.run.smithi077:mon.c> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.c.service 2024-09-13T00:20:56.608 DEBUG:teuthology.orchestra.run.smithi088:mon.b> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.b.service 2024-09-13T00:20:56.611 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T00:20:56.611 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T00:20:57.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:56 smithi077 bash[22596]: cluster 2024-09-13T00:20:55.167281+0000 mgr.y (mgr.14150) 42 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:20:57.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:56 smithi077 bash[22596]: audit 2024-09-13T00:20:55.918038+0000 mon.a (mon.0) 126 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:57.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:56 smithi077 bash[22596]: audit 2024-09-13T00:20:55.919556+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:20:57.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:56 smithi077 bash[22596]: audit 2024-09-13T00:20:55.922840+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:20:57.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:56 smithi077 bash[22596]: audit 2024-09-13T00:20:55.924796+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:20:57.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:56 smithi077 bash[22596]: audit 2024-09-13T00:20:55.933068+0000 mon.a (mon.0) 130 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:20:57.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:56 smithi077 bash[22596]: audit 2024-09-13T00:20:55.936739+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:20:57.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:56 smithi077 bash[22596]: audit 2024-09-13T00:20:55.938535+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:20:57.859 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:20:58.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:57 smithi077 bash[22596]: audit 2024-09-13T00:20:55.907133+0000 mgr.y (mgr.14150) 43 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi077:172.21.15.77=a;smithi077:[v2:172.21.15.77:3301,v1:172.21.15.77:6790]=c;smithi088:172.21.15.88=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:20:58.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:57 smithi077 bash[22596]: cephadm 2024-09-13T00:20:55.912307+0000 mgr.y (mgr.14150) 44 : cephadm [INF] Saving service mon spec with placement smithi077:172.21.15.77=a;smithi077:[v2:172.21.15.77:3301,v1:172.21.15.77:6790]=c;smithi088:172.21.15.88=b;count:3 2024-09-13T00:20:58.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:57 smithi077 bash[22596]: cephadm 2024-09-13T00:20:55.940474+0000 mgr.y (mgr.14150) 45 : cephadm [INF] Deploying daemon mon.b on smithi088 2024-09-13T00:20:59.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:20:58 smithi077 bash[22596]: cluster 2024-09-13T00:20:57.167733+0000 mgr.y (mgr.14150) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:01.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:00 smithi077 bash[22596]: cluster 2024-09-13T00:20:59.168135+0000 mgr.y (mgr.14150) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:01.290 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:01 smithi088 systemd[1]: Started Ceph mon.b for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:21:01.488 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 1 2024-09-13T00:21:01.488 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-13T00:21:01.488 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":1,"fsid":"9218306c-7165-11ef-bceb-c7b262605968","modified":"2024-09-13T00:18:54.475459Z","created":"2024-09-13T00:18:54.475459Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:3300","nonce":0},{"type":"v1","addr":"172.21.15.77:6789","nonce":0}]},"addr":"172.21.15.77:6789/0","public_addr":"172.21.15.77:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-13T00:21:02.454 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:02 smithi077 bash[22596]: cluster 2024-09-13T00:21:01.168554+0000 mgr.y (mgr.14150) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.454 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:02 smithi077 bash[22596]: audit 2024-09-13T00:21:01.173237+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.455 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:02 smithi077 bash[22596]: audit 2024-09-13T00:21:01.179099+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.455 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:02 smithi077 bash[22596]: audit 2024-09-13T00:21:01.187473+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.455 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:02 smithi077 bash[22596]: audit 2024-09-13T00:21:01.188589+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:21:02.455 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:02 smithi077 bash[22596]: audit 2024-09-13T00:21:01.190253+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:02.455 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:02 smithi077 bash[22596]: cephadm 2024-09-13T00:21:01.192078+0000 mgr.y (mgr.14150) 49 : cephadm [INF] Deploying daemon mon.c on smithi077 2024-09-13T00:21:02.455 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:02 smithi077 bash[22596]: audit 2024-09-13T00:21:01.487990+0000 mon.a (mon.0) 138 : audit [DBG] from='client.? 172.21.15.88:0/2168329374' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T00:21:02.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-13T00:21:02.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 0 ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 7 2024-09-13T00:21:02.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 0 pidfile_write: ignore empty --pid-file 2024-09-13T00:21:02.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 0 load: jerasure load: lrc 2024-09-13T00:21:02.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: RocksDB version: 6.15.5 2024-09-13T00:21:02.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T00:21:02.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Compile date Sep 12 2024 2024-09-13T00:21:02.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: DB SUMMARY 2024-09-13T00:21:02.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: DB Session ID: R4PGE53WZHT5TKLLU789 2024-09-13T00:21:02.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: CURRENT file: CURRENT 2024-09-13T00:21:02.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: IDENTITY file: IDENTITY 2024-09-13T00:21:02.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T00:21:02.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-09-13T00:21:02.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-09-13T00:21:02.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.error_if_exists: 0 2024-09-13T00:21:02.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.create_if_missing: 0 2024-09-13T00:21:02.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.paranoid_checks: 1 2024-09-13T00:21:02.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T00:21:02.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.env: 0x555ea4825440 2024-09-13T00:21:02.607 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.fs: Posix File System 2024-09-13T00:21:02.607 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.info_log: 0x555ea6d64ee0 2024-09-13T00:21:02.607 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-13T00:21:02.607 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.statistics: (nil) 2024-09-13T00:21:02.607 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.use_fsync: 0 2024-09-13T00:21:02.607 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.max_log_file_size: 0 2024-09-13T00:21:02.607 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T00:21:02.607 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T00:21:02.608 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-13T00:21:02.608 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-13T00:21:02.608 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.allow_fallocate: 1 2024-09-13T00:21:02.608 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-13T00:21:02.608 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-13T00:21:02.608 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.use_direct_reads: 0 2024-09-13T00:21:02.608 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T00:21:02.608 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.create_missing_column_families: 0 2024-09-13T00:21:02.609 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.db_log_dir: 2024-09-13T00:21:02.609 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2024-09-13T00:21:02.609 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T00:21:02.609 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T00:21:02.609 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T00:21:02.609 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T00:21:02.609 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T00:21:02.609 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T00:21:02.609 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.advise_random_on_open: 1 2024-09-13T00:21:02.609 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-13T00:21:02.609 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.write_buffer_manager: 0x555ea6e227e0 2024-09-13T00:21:02.609 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T00:21:02.610 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T00:21:02.610 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.391+0000 7f97b8ad1900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T00:21:02.610 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T00:21:02.610 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.rate_limiter: (nil) 2024-09-13T00:21:02.610 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T00:21:02.610 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-13T00:21:02.610 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-13T00:21:02.610 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-13T00:21:02.610 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.unordered_write: 0 2024-09-13T00:21:02.610 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T00:21:02.610 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T00:21:02.610 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T00:21:02.611 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T00:21:02.611 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.row_cache: None 2024-09-13T00:21:02.611 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.wal_filter: None 2024-09-13T00:21:02.611 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T00:21:02.611 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-13T00:21:02.611 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.preserve_deletes: 0 2024-09-13T00:21:02.611 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.two_write_queues: 0 2024-09-13T00:21:02.611 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.manual_wal_flush: 0 2024-09-13T00:21:02.611 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.atomic_flush: 0 2024-09-13T00:21:02.611 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T00:21:02.611 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T00:21:02.611 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T00:21:02.612 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.log_readahead_size: 0 2024-09-13T00:21:02.612 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T00:21:02.612 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-13T00:21:02.612 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T00:21:02.612 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T00:21:02.612 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-13T00:21:02.612 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-13T00:21:02.612 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_background_jobs: 2 2024-09-13T00:21:02.612 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_background_compactions: -1 2024-09-13T00:21:02.612 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_subcompactions: 1 2024-09-13T00:21:02.612 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T00:21:02.612 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T00:21:02.613 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T00:21:02.613 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_total_wal_size: 0 2024-09-13T00:21:02.613 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T00:21:02.613 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T00:21:02.613 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T00:21:02.613 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T00:21:02.613 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_open_files: -1 2024-09-13T00:21:02.613 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.bytes_per_sync: 0 2024-09-13T00:21:02.613 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T00:21:02.613 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T00:21:02.613 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-13T00:21:02.613 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_background_flushes: -1 2024-09-13T00:21:02.614 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Compression algorithms supported: 2024-09-13T00:21:02.614 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: kZSTD supported: 0 2024-09-13T00:21:02.614 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: kXpressCompression supported: 0 2024-09-13T00:21:02.614 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: kBZip2Compression supported: 0 2024-09-13T00:21:02.614 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T00:21:02.614 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: kLZ4Compression supported: 1 2024-09-13T00:21:02.614 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: kZlibCompression supported: 1 2024-09-13T00:21:02.614 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-13T00:21:02.614 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: kSnappyCompression supported: 1 2024-09-13T00:21:02.614 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T00:21:02.614 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2024-09-13T00:21:02.614 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T00:21:02.614 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T00:21:02.615 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.merge_operator: 2024-09-13T00:21:02.615 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compaction_filter: None 2024-09-13T00:21:02.615 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compaction_filter_factory: None 2024-09-13T00:21:02.615 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-13T00:21:02.615 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T00:21:02.615 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T00:21:02.615 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555ea6d0da38) 2024-09-13T00:21:02.615 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cache_index_and_filter_blocks: 1 2024-09-13T00:21:02.615 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T00:21:02.615 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T00:21:02.615 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: pin_top_level_index_and_filter: 1 2024-09-13T00:21:02.615 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: index_type: 0 2024-09-13T00:21:02.615 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: data_block_index_type: 0 2024-09-13T00:21:02.616 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: index_shortening: 1 2024-09-13T00:21:02.616 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: data_block_hash_table_util_ratio: 0.750000 2024-09-13T00:21:02.616 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: hash_index_allow_collision: 1 2024-09-13T00:21:02.616 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: checksum: 1 2024-09-13T00:21:02.616 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: no_block_cache: 0 2024-09-13T00:21:02.616 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: block_cache: 0x555ea6d41350 2024-09-13T00:21:02.616 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: block_cache_name: BinnedLRUCache 2024-09-13T00:21:02.616 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: block_cache_options: 2024-09-13T00:21:02.616 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: capacity : 536870912 2024-09-13T00:21:02.616 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: num_shard_bits : 4 2024-09-13T00:21:02.616 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: strict_capacity_limit : 0 2024-09-13T00:21:02.616 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: high_pri_pool_ratio: 0.000 2024-09-13T00:21:02.616 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: block_cache_compressed: (nil) 2024-09-13T00:21:02.617 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: persistent_cache: (nil) 2024-09-13T00:21:02.617 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: block_size: 4096 2024-09-13T00:21:02.617 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: block_size_deviation: 10 2024-09-13T00:21:02.617 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: block_restart_interval: 16 2024-09-13T00:21:02.617 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: index_block_restart_interval: 1 2024-09-13T00:21:02.617 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: metadata_block_size: 4096 2024-09-13T00:21:02.617 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: partition_filters: 0 2024-09-13T00:21:02.617 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: use_delta_encoding: 1 2024-09-13T00:21:02.617 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T00:21:02.617 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: whole_key_filtering: 1 2024-09-13T00:21:02.617 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: verify_compression: 0 2024-09-13T00:21:02.617 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: read_amp_bytes_per_bit: 0 2024-09-13T00:21:02.617 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: format_version: 4 2024-09-13T00:21:02.618 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: enable_index_compression: 1 2024-09-13T00:21:02.618 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: block_align: 0 2024-09-13T00:21:02.618 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-13T00:21:02.618 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-13T00:21:02.618 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compression: NoCompression 2024-09-13T00:21:02.618 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-13T00:21:02.618 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-13T00:21:02.618 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T00:21:02.619 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.num_levels: 7 2024-09-13T00:21:02.620 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T00:21:02.620 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T00:21:02.620 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T00:21:02.620 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T00:21:02.620 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T00:21:02.620 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T00:21:02.620 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T00:21:02.620 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T00:21:02.620 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T00:21:02.620 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T00:21:02.620 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T00:21:02.621 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compression_opts.level: 32767 2024-09-13T00:21:02.621 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-13T00:21:02.621 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T00:21:02.621 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T00:21:02.621 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T00:21:02.621 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compression_opts.enabled: false 2024-09-13T00:21:02.621 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T00:21:02.621 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T00:21:02.621 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T00:21:02.621 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-13T00:21:02.621 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T00:21:02.621 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T00:21:02.621 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T00:21:02.622 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T00:21:02.622 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T00:21:02.622 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T00:21:02.622 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T00:21:02.622 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T00:21:02.622 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T00:21:02.622 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T00:21:02.622 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T00:21:02.622 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T00:21:02.622 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T00:21:02.622 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.arena_block_size: 4194304 2024-09-13T00:21:02.622 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T00:21:02.622 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T00:21:02.623 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T00:21:02.623 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-13T00:21:02.623 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T00:21:02.623 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T00:21:02.623 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T00:21:02.623 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T00:21:02.623 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T00:21:02.623 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T00:21:02.623 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T00:21:02.623 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T00:21:02.623 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T00:21:02.623 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T00:21:02.623 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T00:21:02.623 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.inplace_update_support: 0 2024-09-13T00:21:02.624 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T00:21:02.624 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T00:21:02.624 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T00:21:02.624 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T00:21:02.624 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.bloom_locality: 0 2024-09-13T00:21:02.624 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.max_successive_merges: 0 2024-09-13T00:21:02.624 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T00:21:02.624 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-13T00:21:02.624 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.force_consistency_checks: 1 2024-09-13T00:21:02.624 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-13T00:21:02.624 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.ttl: 2592000 2024-09-13T00:21:02.624 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T00:21:02.625 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.enable_blob_files: false 2024-09-13T00:21:02.625 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.min_blob_size: 0 2024-09-13T00:21:02.625 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.blob_file_size: 268435456 2024-09-13T00:21:02.625 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T00:21:02.625 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T00:21:02.625 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T00:21:02.625 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T00:21:02.625 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-13T00:21:02.625 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-13T00:21:02.625 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726186862399610, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T00:21:02.625 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-13T00:21:02.625 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 3 rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T00:21:02.625 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726186862400102, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1571, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726186862, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "d59b9ce4-ada9-4e34-9af2-725521cf0dbe", "db_session_id": "R4PGE53WZHT5TKLLU789"}} 2024-09-13T00:21:02.626 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-13T00:21:02.626 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726186862400638, "job": 1, "event": "recovery_finished"} 2024-09-13T00:21:02.626 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.395+0000 7f97b8ad1900 4 rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T00:21:02.626 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.399+0000 7f97b8ad1900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x555ea6d76700 2024-09-13T00:21:02.626 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.399+0000 7f97b8ad1900 4 rocksdb: DB pointer 0x555ea6d44000 2024-09-13T00:21:02.626 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.399+0000 7f97ae89d640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T00:21:02.626 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.399+0000 7f97ae89d640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T00:21:02.626 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: ** DB Stats ** 2024-09-13T00:21:02.626 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:21:02.626 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T00:21:02.626 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T00:21:02.626 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T00:21:02.627 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T00:21:02.627 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T00:21:02.627 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T00:21:02.627 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: ** Compaction Stats [default] ** 2024-09-13T00:21:02.627 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T00:21:02.627 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:21:02.627 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2024-09-13T00:21:02.627 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2024-09-13T00:21:02.627 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2024-09-13T00:21:02.627 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: ** Compaction Stats [default] ** 2024-09-13T00:21:02.627 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T00:21:02.628 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:21:02.628 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2024-09-13T00:21:02.628 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:21:02.628 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T00:21:02.628 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T00:21:02.628 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T00:21:02.628 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T00:21:02.628 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T00:21:02.628 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Cumulative compaction: 0.00 GB write, 0.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:21:02.628 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Interval compaction: 0.00 GB write, 0.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:21:02.628 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T00:21:02.628 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T00:21:02.628 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: ** Compaction Stats [default] ** 2024-09-13T00:21:02.629 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T00:21:02.629 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:21:02.629 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2024-09-13T00:21:02.629 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2024-09-13T00:21:02.629 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T00:21:02.629 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: ** Compaction Stats [default] ** 2024-09-13T00:21:02.629 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T00:21:02.629 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:21:02.629 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.2 0.00 0.00 1 0.000 0 0 2024-09-13T00:21:02.629 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:21:02.629 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T00:21:02.629 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T00:21:02.629 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T00:21:02.630 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T00:21:02.630 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T00:21:02.630 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Cumulative compaction: 0.00 GB write, 0.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:21:02.630 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:21:02.630 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T00:21:02.630 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T00:21:02.630 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.399+0000 7f97b8ad1900 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2024-09-13T00:21:02.630 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.399+0000 7f97b8ad1900 0 using public_addr v2:172.21.15.88:0/0 -> [v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0] 2024-09-13T00:21:02.630 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.399+0000 7f97b8ad1900 0 starting mon.b rank -1 at public addrs [v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0] at bind addrs [v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:21:02.630 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.399+0000 7f97b8ad1900 1 mon.b@-1(???) e0 preinit fsid 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:21:02.630 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.427+0000 7f97b18a3640 0 mon.b@-1(synchronizing).mds e1 new map 2024-09-13T00:21:02.630 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.427+0000 7f97b18a3640 0 mon.b@-1(synchronizing).mds e1 print_map 2024-09-13T00:21:02.630 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: e1 2024-09-13T00:21:02.631 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T00:21:02.631 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T00:21:02.631 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: legacy client fscid: -1 2024-09-13T00:21:02.631 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: 2024-09-13T00:21:02.631 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: No filesystems configured 2024-09-13T00:21:02.631 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.427+0000 7f97b18a3640 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T00:21:02.631 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.427+0000 7f97b18a3640 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T00:21:02.631 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.427+0000 7f97b18a3640 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T00:21:02.631 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.427+0000 7f97b18a3640 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T00:21:02.631 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.427+0000 7f97b18a3640 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T00:21:02.631 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.427+0000 7f97b18a3640 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T00:21:02.631 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.427+0000 7f97b18a3640 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T00:21:02.631 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.427+0000 7f97b18a3640 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T00:21:02.632 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.427+0000 7f97b18a3640 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T00:21:02.632 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.427+0000 7f97b18a3640 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T00:21:02.632 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:27.161665+0000 mgr.y (mgr.14150) 20 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.632 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:29.162082+0000 mgr.y (mgr.14150) 21 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.632 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:31.162512+0000 mgr.y (mgr.14150) 22 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.632 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:31.840733+0000 mgr.y (mgr.14150) 23 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi088", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:21:02.632 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cephadm 2024-09-13T00:20:32.826470+0000 mgr.y (mgr.14150) 24 : cephadm [INF] Deploying cephadm binary to smithi088 2024-09-13T00:21:02.632 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:33.162896+0000 mgr.y (mgr.14150) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.632 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:35.084212+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.632 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cephadm 2024-09-13T00:20:35.085082+0000 mgr.y (mgr.14150) 26 : cephadm [INF] Added host smithi088 2024-09-13T00:21:02.632 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:35.085935+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:02.632 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:35.163166+0000 mgr.y (mgr.14150) 27 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.632 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:35.342903+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.633 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:36.611783+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.633 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:37.399022+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.633 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:37.163481+0000 mgr.y (mgr.14150) 28 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.633 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:39.163888+0000 mgr.y (mgr.14150) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.633 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:41.164285+0000 mgr.y (mgr.14150) 30 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.633 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:42.590060+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.633 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:42.597631+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.633 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:42.604063+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.633 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:42.609572+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.633 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:42.610990+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi088", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:21:02.633 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:42.613016+0000 mon.a (mon.0) 118 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:02.633 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:42.614708+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:02.633 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cephadm 2024-09-13T00:20:42.617126+0000 mgr.y (mgr.14150) 31 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:21:02.634 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cephadm 2024-09-13T00:20:42.792296+0000 mgr.y (mgr.14150) 32 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:21:02.634 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:43.183730+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.634 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:43.186808+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.634 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:43.190285+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.634 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cephadm 2024-09-13T00:20:42.939353+0000 mgr.y (mgr.14150) 33 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:21:02.634 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cephadm 2024-09-13T00:20:43.075392+0000 mgr.y (mgr.14150) 34 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:21:02.634 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:43.164627+0000 mgr.y (mgr.14150) 35 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.634 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:43.357653+0000 mgr.y (mgr.14150) 36 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:21:02.634 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:45.165090+0000 mgr.y (mgr.14150) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.634 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:47.165604+0000 mgr.y (mgr.14150) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.634 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:49.166054+0000 mgr.y (mgr.14150) 39 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.634 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:51.541441+0000 mon.a (mon.0) 123 : audit [INF] from='client.? 172.21.15.77:0/738112169' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T00:21:02.635 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:51.166462+0000 mgr.y (mgr.14150) 40 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.635 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:51.644237+0000 mon.a (mon.0) 124 : audit [INF] from='client.? 172.21.15.77:0/738112169' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T00:21:02.635 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:51.644344+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T00:21:02.635 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:53.166873+0000 mgr.y (mgr.14150) 41 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.635 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:55.167281+0000 mgr.y (mgr.14150) 42 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.635 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:55.918038+0000 mon.a (mon.0) 126 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.635 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:55.919556+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:02.635 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:55.922840+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:02.635 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:55.924796+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:02.635 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:55.933068+0000 mon.a (mon.0) 130 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.635 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:55.936739+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:21:02.635 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:55.938535+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:02.636 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:20:55.907133+0000 mgr.y (mgr.14150) 43 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi077:172.21.15.77=a;smithi077:[v2:172.21.15.77:3301,v1:172.21.15.77:6790]=c;smithi088:172.21.15.88=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:21:02.636 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cephadm 2024-09-13T00:20:55.912307+0000 mgr.y (mgr.14150) 44 : cephadm [INF] Saving service mon spec with placement smithi077:172.21.15.77=a;smithi077:[v2:172.21.15.77:3301,v1:172.21.15.77:6790]=c;smithi088:172.21.15.88=b;count:3 2024-09-13T00:21:02.636 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cephadm 2024-09-13T00:20:55.940474+0000 mgr.y (mgr.14150) 45 : cephadm [INF] Deploying daemon mon.b on smithi088 2024-09-13T00:21:02.636 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:57.167733+0000 mgr.y (mgr.14150) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.636 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:20:59.168135+0000 mgr.y (mgr.14150) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.636 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cluster 2024-09-13T00:21:01.168554+0000 mgr.y (mgr.14150) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:02.636 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:21:01.173237+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.636 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:21:01.179099+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.636 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:21:01.187473+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:02.636 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:21:01.188589+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:21:02.636 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:21:01.190253+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:02.636 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: cephadm 2024-09-13T00:21:01.192078+0000 mgr.y (mgr.14150) 49 : cephadm [INF] Deploying daemon mon.c on smithi077 2024-09-13T00:21:02.637 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: audit 2024-09-13T00:21:01.487990+0000 mon.a (mon.0) 138 : audit [DBG] from='client.? 172.21.15.88:0/2168329374' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T00:21:02.637 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:02 smithi088 bash[24895]: debug 2024-09-13T00:21:02.431+0000 7f97b18a3640 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-13T00:21:03.188 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-13T00:21:03.189 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-13T00:21:04.816 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:04 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:21:04.816 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:21:04 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:21:05.159 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:05 smithi077 systemd[1]: Started Ceph mon.c for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:21:05.159 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:04 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:21:05.160 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:21:04 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:21:06.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.092+0000 7f3d83d83900 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-13T00:21:06.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.092+0000 7f3d83d83900 0 ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable), process ceph-mon, pid 7 2024-09-13T00:21:06.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.092+0000 7f3d83d83900 0 pidfile_write: ignore empty --pid-file 2024-09-13T00:21:06.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 0 load: jerasure load: lrc 2024-09-13T00:21:06.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: RocksDB version: 6.15.5 2024-09-13T00:21:06.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-09-13T00:21:06.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Compile date Sep 12 2024 2024-09-13T00:21:06.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: DB SUMMARY 2024-09-13T00:21:06.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: DB Session ID: KZ0OTP2X7BTYEQ519UR7 2024-09-13T00:21:06.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: CURRENT file: CURRENT 2024-09-13T00:21:06.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: IDENTITY file: IDENTITY 2024-09-13T00:21:06.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2024-09-13T00:21:06.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-13T00:21:06.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 511 ; 2024-09-13T00:21:06.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.error_if_exists: 0 2024-09-13T00:21:06.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.create_if_missing: 0 2024-09-13T00:21:06.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.paranoid_checks: 1 2024-09-13T00:21:06.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T00:21:06.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.env: 0x5611267ed440 2024-09-13T00:21:06.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.fs: Posix File System 2024-09-13T00:21:06.284 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.info_log: 0x5611276d6ee0 2024-09-13T00:21:06.284 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-13T00:21:06.284 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.statistics: (nil) 2024-09-13T00:21:06.284 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.use_fsync: 0 2024-09-13T00:21:06.284 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_log_file_size: 0 2024-09-13T00:21:06.284 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T00:21:06.284 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T00:21:06.285 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-13T00:21:06.285 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-13T00:21:06.285 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.allow_fallocate: 1 2024-09-13T00:21:06.285 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-13T00:21:06.285 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-13T00:21:06.285 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.use_direct_reads: 0 2024-09-13T00:21:06.285 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T00:21:06.285 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.create_missing_column_families: 0 2024-09-13T00:21:06.285 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.db_log_dir: 2024-09-13T00:21:06.285 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2024-09-13T00:21:06.286 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T00:21:06.286 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T00:21:06.286 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T00:21:06.286 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T00:21:06.286 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T00:21:06.286 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T00:21:06.286 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.advise_random_on_open: 1 2024-09-13T00:21:06.286 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-13T00:21:06.286 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.write_buffer_manager: 0x561127794810 2024-09-13T00:21:06.286 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T00:21:06.286 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-09-13T00:21:06.286 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T00:21:06.286 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T00:21:06.287 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.rate_limiter: (nil) 2024-09-13T00:21:06.287 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T00:21:06.287 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-13T00:21:06.287 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-13T00:21:06.287 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-13T00:21:06.287 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.unordered_write: 0 2024-09-13T00:21:06.287 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T00:21:06.287 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T00:21:06.287 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T00:21:06.287 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T00:21:06.287 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.row_cache: None 2024-09-13T00:21:06.287 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.wal_filter: None 2024-09-13T00:21:06.287 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T00:21:06.288 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-13T00:21:06.288 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.preserve_deletes: 0 2024-09-13T00:21:06.288 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.two_write_queues: 0 2024-09-13T00:21:06.288 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.manual_wal_flush: 0 2024-09-13T00:21:06.288 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.atomic_flush: 0 2024-09-13T00:21:06.288 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T00:21:06.288 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T00:21:06.288 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T00:21:06.288 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.log_readahead_size: 0 2024-09-13T00:21:06.288 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T00:21:06.288 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-13T00:21:06.288 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T00:21:06.288 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T00:21:06.289 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-13T00:21:06.289 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-13T00:21:06.289 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_background_jobs: 2 2024-09-13T00:21:06.289 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_background_compactions: -1 2024-09-13T00:21:06.289 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_subcompactions: 1 2024-09-13T00:21:06.289 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T00:21:06.289 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T00:21:06.289 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T00:21:06.289 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_total_wal_size: 0 2024-09-13T00:21:06.289 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T00:21:06.289 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T00:21:06.289 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T00:21:06.289 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T00:21:06.290 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_open_files: -1 2024-09-13T00:21:06.290 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.bytes_per_sync: 0 2024-09-13T00:21:06.290 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T00:21:06.290 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T00:21:06.290 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-13T00:21:06.290 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_background_flushes: -1 2024-09-13T00:21:06.290 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Compression algorithms supported: 2024-09-13T00:21:06.290 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: kZSTD supported: 0 2024-09-13T00:21:06.290 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: kXpressCompression supported: 0 2024-09-13T00:21:06.290 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: kBZip2Compression supported: 0 2024-09-13T00:21:06.290 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T00:21:06.290 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: kLZ4Compression supported: 1 2024-09-13T00:21:06.290 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: kZlibCompression supported: 1 2024-09-13T00:21:06.290 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-13T00:21:06.291 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: kSnappyCompression supported: 1 2024-09-13T00:21:06.291 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T00:21:06.291 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2024-09-13T00:21:06.291 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-09-13T00:21:06.291 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T00:21:06.291 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.merge_operator: 2024-09-13T00:21:06.293 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compaction_filter: None 2024-09-13T00:21:06.293 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compaction_filter_factory: None 2024-09-13T00:21:06.293 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-13T00:21:06.293 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T00:21:06.293 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T00:21:06.293 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56112767fa38) 2024-09-13T00:21:06.293 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cache_index_and_filter_blocks: 1 2024-09-13T00:21:06.294 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T00:21:06.294 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T00:21:06.294 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: pin_top_level_index_and_filter: 1 2024-09-13T00:21:06.294 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: index_type: 0 2024-09-13T00:21:06.294 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: data_block_index_type: 0 2024-09-13T00:21:06.294 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: index_shortening: 1 2024-09-13T00:21:06.294 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: data_block_hash_table_util_ratio: 0.750000 2024-09-13T00:21:06.294 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: hash_index_allow_collision: 1 2024-09-13T00:21:06.294 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: checksum: 1 2024-09-13T00:21:06.295 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: no_block_cache: 0 2024-09-13T00:21:06.295 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: block_cache: 0x5611276b3350 2024-09-13T00:21:06.295 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: block_cache_name: BinnedLRUCache 2024-09-13T00:21:06.295 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: block_cache_options: 2024-09-13T00:21:06.295 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: capacity : 536870912 2024-09-13T00:21:06.295 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: num_shard_bits : 4 2024-09-13T00:21:06.295 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: strict_capacity_limit : 0 2024-09-13T00:21:06.295 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: high_pri_pool_ratio: 0.000 2024-09-13T00:21:06.295 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: block_cache_compressed: (nil) 2024-09-13T00:21:06.295 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: persistent_cache: (nil) 2024-09-13T00:21:06.295 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: block_size: 4096 2024-09-13T00:21:06.295 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: block_size_deviation: 10 2024-09-13T00:21:06.295 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: block_restart_interval: 16 2024-09-13T00:21:06.296 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: index_block_restart_interval: 1 2024-09-13T00:21:06.296 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: metadata_block_size: 4096 2024-09-13T00:21:06.296 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: partition_filters: 0 2024-09-13T00:21:06.296 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: use_delta_encoding: 1 2024-09-13T00:21:06.296 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: filter_policy: rocksdb.BuiltinBloomFilter 2024-09-13T00:21:06.296 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: whole_key_filtering: 1 2024-09-13T00:21:06.296 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: verify_compression: 0 2024-09-13T00:21:06.296 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: read_amp_bytes_per_bit: 0 2024-09-13T00:21:06.296 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: format_version: 4 2024-09-13T00:21:06.296 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: enable_index_compression: 1 2024-09-13T00:21:06.296 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: block_align: 0 2024-09-13T00:21:06.296 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-13T00:21:06.296 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-13T00:21:06.297 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compression: NoCompression 2024-09-13T00:21:06.297 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-13T00:21:06.297 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-13T00:21:06.297 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T00:21:06.297 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.num_levels: 7 2024-09-13T00:21:06.297 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T00:21:06.297 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T00:21:06.297 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T00:21:06.297 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T00:21:06.297 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T00:21:06.297 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T00:21:06.297 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T00:21:06.297 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T00:21:06.298 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T00:21:06.298 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T00:21:06.298 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T00:21:06.298 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compression_opts.level: 32767 2024-09-13T00:21:06.298 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-13T00:21:06.298 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T00:21:06.298 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T00:21:06.298 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T00:21:06.298 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compression_opts.enabled: false 2024-09-13T00:21:06.298 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T00:21:06.298 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T00:21:06.298 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T00:21:06.298 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-13T00:21:06.299 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T00:21:06.299 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T00:21:06.299 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T00:21:06.299 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T00:21:06.299 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T00:21:06.299 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T00:21:06.299 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T00:21:06.299 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T00:21:06.299 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T00:21:06.299 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T00:21:06.299 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T00:21:06.299 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T00:21:06.299 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T00:21:06.300 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.arena_block_size: 4194304 2024-09-13T00:21:06.300 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T00:21:06.300 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T00:21:06.300 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-09-13T00:21:06.300 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-13T00:21:06.300 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T00:21:06.300 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T00:21:06.300 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T00:21:06.301 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T00:21:06.301 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T00:21:06.301 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T00:21:06.301 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T00:21:06.301 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T00:21:06.301 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T00:21:06.301 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T00:21:06.301 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T00:21:06.301 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.inplace_update_support: 0 2024-09-13T00:21:06.301 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T00:21:06.301 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T00:21:06.301 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T00:21:06.301 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T00:21:06.302 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.bloom_locality: 0 2024-09-13T00:21:06.302 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.max_successive_merges: 0 2024-09-13T00:21:06.302 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T00:21:06.302 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-13T00:21:06.302 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.force_consistency_checks: 1 2024-09-13T00:21:06.302 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-13T00:21:06.302 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.ttl: 2592000 2024-09-13T00:21:06.302 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T00:21:06.302 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.enable_blob_files: false 2024-09-13T00:21:06.302 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.min_blob_size: 0 2024-09-13T00:21:06.302 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.blob_file_size: 268435456 2024-09-13T00:21:06.302 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T00:21:06.302 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T00:21:06.303 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T00:21:06.303 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T00:21:06.303 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2024-09-13T00:21:06.303 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 7 2024-09-13T00:21:06.303 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726186866102054, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-13T00:21:06.303 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2024-09-13T00:21:06.303 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 3 rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-09-13T00:21:06.303 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.096+0000 7f3d83d83900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726186866102465, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1571, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1726186866, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "a5390e97-99af-4c74-8ca6-02517b6d2652", "db_session_id": "KZ0OTP2X7BTYEQ519UR7"}} 2024-09-13T00:21:06.303 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.100+0000 7f3d83d83900 4 rocksdb: [db/version_set.cc:4082] Creating manifest 9 2024-09-13T00:21:06.303 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.100+0000 7f3d83d83900 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726186866102932, "job": 1, "event": "recovery_finished"} 2024-09-13T00:21:06.303 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.100+0000 7f3d83d83900 4 rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T00:21:06.303 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.100+0000 7f3d83d83900 4 rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5611276e8700 2024-09-13T00:21:06.304 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.100+0000 7f3d83d83900 4 rocksdb: DB pointer 0x5611276b6000 2024-09-13T00:21:06.304 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.100+0000 7f3d79b4f640 4 rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-09-13T00:21:06.304 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.100+0000 7f3d79b4f640 4 rocksdb: [db/db_impl/db_impl.cc:903] 2024-09-13T00:21:06.304 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: ** DB Stats ** 2024-09-13T00:21:06.304 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:21:06.304 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T00:21:06.304 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T00:21:06.304 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T00:21:06.304 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T00:21:06.304 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-09-13T00:21:06.304 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T00:21:06.304 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: ** Compaction Stats [default] ** 2024-09-13T00:21:06.304 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T00:21:06.305 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:21:06.305 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-09-13T00:21:06.305 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-09-13T00:21:06.305 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-09-13T00:21:06.305 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: ** Compaction Stats [default] ** 2024-09-13T00:21:06.305 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T00:21:06.305 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:21:06.305 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-09-13T00:21:06.305 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:21:06.305 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T00:21:06.305 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T00:21:06.305 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T00:21:06.305 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T00:21:06.306 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T00:21:06.306 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Cumulative compaction: 0.00 GB write, 0.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:21:06.306 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Interval compaction: 0.00 GB write, 0.44 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:21:06.306 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T00:21:06.306 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T00:21:06.306 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: ** Compaction Stats [default] ** 2024-09-13T00:21:06.306 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T00:21:06.306 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:21:06.306 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: L0 1/0 1.53 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-09-13T00:21:06.306 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Sum 1/0 1.53 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-09-13T00:21:06.306 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-09-13T00:21:06.306 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: ** Compaction Stats [default] ** 2024-09-13T00:21:06.306 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-09-13T00:21:06.307 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:21:06.307 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.8 0.00 0.00 1 0.000 0 0 2024-09-13T00:21:06.307 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:21:06.307 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T00:21:06.307 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T00:21:06.307 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T00:21:06.307 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T00:21:06.307 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T00:21:06.307 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Cumulative compaction: 0.00 GB write, 0.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:21:06.307 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:21:06.307 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T00:21:06.307 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T00:21:06.307 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.100+0000 7f3d83d83900 0 mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-13T00:21:06.308 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.100+0000 7f3d83d83900 0 using public_addrv [v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0] 2024-09-13T00:21:06.308 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.100+0000 7f3d83d83900 0 starting mon.c rank -1 at public addrs [v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0] at bind addrs [v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:21:06.308 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.100+0000 7f3d83d83900 1 mon.c@-1(???) e0 preinit fsid 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:21:06.308 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.120+0000 7f3d7cb55640 0 mon.c@-1(synchronizing).mds e1 new map 2024-09-13T00:21:06.308 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.120+0000 7f3d7cb55640 0 mon.c@-1(synchronizing).mds e1 print_map 2024-09-13T00:21:06.308 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: e1 2024-09-13T00:21:06.308 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T00:21:06.308 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T00:21:06.308 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: legacy client fscid: -1 2024-09-13T00:21:06.308 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: 2024-09-13T00:21:06.308 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: No filesystems configured 2024-09-13T00:21:06.308 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.120+0000 7f3d7cb55640 1 mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-13T00:21:06.308 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.120+0000 7f3d7cb55640 1 mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-13T00:21:06.309 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.120+0000 7f3d7cb55640 1 mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-13T00:21:06.309 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.120+0000 7f3d7cb55640 1 mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-13T00:21:06.309 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.120+0000 7f3d7cb55640 1 mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-13T00:21:06.309 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.120+0000 7f3d7cb55640 1 mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-13T00:21:06.309 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.120+0000 7f3d7cb55640 0 mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-13T00:21:06.309 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.120+0000 7f3d7cb55640 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T00:21:06.309 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.120+0000 7f3d7cb55640 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T00:21:06.309 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.120+0000 7f3d7cb55640 0 mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-13T00:21:06.309 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:27.161665+0000 mgr.y (mgr.14150) 20 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.309 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:29.162082+0000 mgr.y (mgr.14150) 21 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.309 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:31.162512+0000 mgr.y (mgr.14150) 22 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.309 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:31.840733+0000 mgr.y (mgr.14150) 23 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi088", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:21:06.309 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cephadm 2024-09-13T00:20:32.826470+0000 mgr.y (mgr.14150) 24 : cephadm [INF] Deploying cephadm binary to smithi088 2024-09-13T00:21:06.309 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:33.162896+0000 mgr.y (mgr.14150) 25 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.310 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:35.084212+0000 mon.a (mon.0) 108 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:06.310 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cephadm 2024-09-13T00:20:35.085082+0000 mgr.y (mgr.14150) 26 : cephadm [INF] Added host smithi088 2024-09-13T00:21:06.310 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:35.085935+0000 mon.a (mon.0) 109 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:06.310 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:35.163166+0000 mgr.y (mgr.14150) 27 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.310 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:35.342903+0000 mon.a (mon.0) 110 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:06.310 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:36.611783+0000 mon.a (mon.0) 111 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:06.310 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:37.399022+0000 mon.a (mon.0) 112 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:06.310 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:37.163481+0000 mgr.y (mgr.14150) 28 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.310 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:39.163888+0000 mgr.y (mgr.14150) 29 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.310 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:41.164285+0000 mgr.y (mgr.14150) 30 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.310 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:42.590060+0000 mon.a (mon.0) 113 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:06.310 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:42.597631+0000 mon.a (mon.0) 114 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:06.310 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:42.604063+0000 mon.a (mon.0) 115 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:06.310 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:42.609572+0000 mon.a (mon.0) 116 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:06.311 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:42.610990+0000 mon.a (mon.0) 117 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi088", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:21:06.311 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:42.613016+0000 mon.a (mon.0) 118 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:06.311 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:42.614708+0000 mon.a (mon.0) 119 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:06.311 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cephadm 2024-09-13T00:20:42.617126+0000 mgr.y (mgr.14150) 31 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:21:06.311 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cephadm 2024-09-13T00:20:42.792296+0000 mgr.y (mgr.14150) 32 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:21:06.311 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:43.183730+0000 mon.a (mon.0) 120 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:06.311 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:43.186808+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:06.311 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:43.190285+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:06.311 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cephadm 2024-09-13T00:20:42.939353+0000 mgr.y (mgr.14150) 33 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:21:06.311 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cephadm 2024-09-13T00:20:43.075392+0000 mgr.y (mgr.14150) 34 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:21:06.311 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:43.164627+0000 mgr.y (mgr.14150) 35 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.311 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:43.357653+0000 mgr.y (mgr.14150) 36 : audit [DBG] from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:21:06.312 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:45.165090+0000 mgr.y (mgr.14150) 37 : cluster [DBG] pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.312 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:47.165604+0000 mgr.y (mgr.14150) 38 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.312 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:49.166054+0000 mgr.y (mgr.14150) 39 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.312 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:51.541441+0000 mon.a (mon.0) 123 : audit [INF] from='client.? 172.21.15.77:0/738112169' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-13T00:21:06.312 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:51.166462+0000 mgr.y (mgr.14150) 40 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.312 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:51.644237+0000 mon.a (mon.0) 124 : audit [INF] from='client.? 172.21.15.77:0/738112169' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-13T00:21:06.312 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:51.644344+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T00:21:06.312 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:53.166873+0000 mgr.y (mgr.14150) 41 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.312 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:55.167281+0000 mgr.y (mgr.14150) 42 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.312 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:55.918038+0000 mon.a (mon.0) 126 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:06.313 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:55.919556+0000 mon.a (mon.0) 127 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:06.313 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:55.922840+0000 mon.a (mon.0) 128 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:06.313 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:55.924796+0000 mon.a (mon.0) 129 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:06.313 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:55.933068+0000 mon.a (mon.0) 130 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:06.313 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:55.936739+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:21:06.313 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:55.938535+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:06.313 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:20:55.907133+0000 mgr.y (mgr.14150) 43 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi077:172.21.15.77=a;smithi077:[v2:172.21.15.77:3301,v1:172.21.15.77:6790]=c;smithi088:172.21.15.88=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:21:06.313 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cephadm 2024-09-13T00:20:55.912307+0000 mgr.y (mgr.14150) 44 : cephadm [INF] Saving service mon spec with placement smithi077:172.21.15.77=a;smithi077:[v2:172.21.15.77:3301,v1:172.21.15.77:6790]=c;smithi088:172.21.15.88=b;count:3 2024-09-13T00:21:06.313 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cephadm 2024-09-13T00:20:55.940474+0000 mgr.y (mgr.14150) 45 : cephadm [INF] Deploying daemon mon.b on smithi088 2024-09-13T00:21:06.313 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:57.167733+0000 mgr.y (mgr.14150) 46 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.313 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:20:59.168135+0000 mgr.y (mgr.14150) 47 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.313 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cluster 2024-09-13T00:21:01.168554+0000 mgr.y (mgr.14150) 48 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:06.313 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:21:01.173237+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:06.313 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:21:01.179099+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:06.314 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:21:01.187473+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:06.314 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:21:01.188589+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:21:06.314 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:21:01.190253+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:06.314 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: cephadm 2024-09-13T00:21:01.192078+0000 mgr.y (mgr.14150) 49 : cephadm [INF] Deploying daemon mon.c on smithi077 2024-09-13T00:21:06.314 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: audit 2024-09-13T00:21:01.487990+0000 mon.a (mon.0) 138 : audit [DBG] from='client.? 172.21.15.88:0/2168329374' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T00:21:06.314 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:06 smithi077 bash[32040]: debug 2024-09-13T00:21:06.124+0000 7f3d7cb55640 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-13T00:21:06.893 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T00:21:07.836 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: audit 2024-09-13T00:21:02.444446+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:21:07.836 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: audit 2024-09-13T00:21:02.444579+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:07.836 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: cluster 2024-09-13T00:21:02.444741+0000 mon.a (mon.0) 142 : cluster [INF] mon.a calling monitor election 2024-09-13T00:21:07.836 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: cluster 2024-09-13T00:21:03.169114+0000 mgr.y (mgr.14150) 50 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:07.836 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: audit 2024-09-13T00:21:03.439314+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:07.836 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: audit 2024-09-13T00:21:04.439876+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:07.837 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: cluster 2024-09-13T00:21:04.441896+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2024-09-13T00:21:07.837 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: cluster 2024-09-13T00:21:05.169438+0000 mgr.y (mgr.14150) 51 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:07.837 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: audit 2024-09-13T00:21:05.439896+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:07.837 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: audit 2024-09-13T00:21:06.132836+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:07.837 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: audit 2024-09-13T00:21:06.440016+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:07.837 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: audit 2024-09-13T00:21:07.133608+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:07.837 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: audit 2024-09-13T00:21:07.440174+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:07.837 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: cluster 2024-09-13T00:21:07.471546+0000 mon.a (mon.0) 150 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T00:21:07.837 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: cluster 2024-09-13T00:21:07.481978+0000 mon.a (mon.0) 151 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:21:07.837 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: cluster 2024-09-13T00:21:07.499379+0000 mon.a (mon.0) 152 : cluster [DBG] fsmap 2024-09-13T00:21:07.837 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: cluster 2024-09-13T00:21:07.499428+0000 mon.a (mon.0) 153 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T00:21:07.837 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: cluster 2024-09-13T00:21:07.499680+0000 mon.a (mon.0) 154 : cluster [DBG] mgrmap e13: y(active, since 72s) 2024-09-13T00:21:07.837 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: cluster 2024-09-13T00:21:07.499914+0000 mon.a (mon.0) 155 : cluster [INF] overall HEALTH_OK 2024-09-13T00:21:07.838 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: audit 2024-09-13T00:21:07.506401+0000 mon.a (mon.0) 156 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:07.838 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: audit 2024-09-13T00:21:07.513833+0000 mon.a (mon.0) 157 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:07.838 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: audit 2024-09-13T00:21:07.521251+0000 mon.a (mon.0) 158 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:07.838 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:07 smithi088 bash[24895]: audit 2024-09-13T00:21:07.528163+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:08.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: audit 2024-09-13T00:21:02.444446+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:21:08.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: audit 2024-09-13T00:21:02.444579+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:08.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: cluster 2024-09-13T00:21:02.444741+0000 mon.a (mon.0) 142 : cluster [INF] mon.a calling monitor election 2024-09-13T00:21:08.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: cluster 2024-09-13T00:21:03.169114+0000 mgr.y (mgr.14150) 50 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:08.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: audit 2024-09-13T00:21:03.439314+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:08.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: audit 2024-09-13T00:21:04.439876+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:08.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: cluster 2024-09-13T00:21:04.441896+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2024-09-13T00:21:08.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: cluster 2024-09-13T00:21:05.169438+0000 mgr.y (mgr.14150) 51 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:08.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: audit 2024-09-13T00:21:05.439896+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:08.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: audit 2024-09-13T00:21:06.132836+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:08.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: audit 2024-09-13T00:21:06.440016+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:08.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: audit 2024-09-13T00:21:07.133608+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:08.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: audit 2024-09-13T00:21:07.440174+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:08.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: cluster 2024-09-13T00:21:07.471546+0000 mon.a (mon.0) 150 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T00:21:08.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: cluster 2024-09-13T00:21:07.481978+0000 mon.a (mon.0) 151 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:21:08.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: cluster 2024-09-13T00:21:07.499379+0000 mon.a (mon.0) 152 : cluster [DBG] fsmap 2024-09-13T00:21:08.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: cluster 2024-09-13T00:21:07.499428+0000 mon.a (mon.0) 153 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T00:21:08.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: cluster 2024-09-13T00:21:07.499680+0000 mon.a (mon.0) 154 : cluster [DBG] mgrmap e13: y(active, since 72s) 2024-09-13T00:21:08.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: cluster 2024-09-13T00:21:07.499914+0000 mon.a (mon.0) 155 : cluster [INF] overall HEALTH_OK 2024-09-13T00:21:08.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: audit 2024-09-13T00:21:07.506401+0000 mon.a (mon.0) 156 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:08.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: audit 2024-09-13T00:21:07.513833+0000 mon.a (mon.0) 157 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:08.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: audit 2024-09-13T00:21:07.521251+0000 mon.a (mon.0) 158 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:08.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:07 smithi077 bash[22596]: audit 2024-09-13T00:21:07.528163+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:08.781 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:21:08 smithi077 bash[22845]: debug 2024-09-13T00:21:08.436+0000 7fa1cb86c640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-13T00:21:13.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: audit 2024-09-13T00:21:08.140862+0000 mon.a (mon.0) 162 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:21:13.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: cluster 2024-09-13T00:21:08.141358+0000 mon.a (mon.0) 163 : cluster [INF] mon.a calling monitor election 2024-09-13T00:21:13.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: cluster 2024-09-13T00:21:08.142314+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-13T00:21:13.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: audit 2024-09-13T00:21:08.159959+0000 mon.a (mon.0) 164 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:13.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: audit 2024-09-13T00:21:08.160291+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: audit 2024-09-13T00:21:09.134372+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: cluster 2024-09-13T00:21:09.170144+0000 mgr.y (mgr.14150) 53 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:13.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: audit 2024-09-13T00:21:10.134536+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: cluster 2024-09-13T00:21:10.136582+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-13T00:21:13.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: audit 2024-09-13T00:21:11.134683+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: cluster 2024-09-13T00:21:11.170555+0000 mgr.y (mgr.14150) 54 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:13.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: audit 2024-09-13T00:21:12.134602+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: audit 2024-09-13T00:21:13.134773+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: cluster 2024-09-13T00:21:13.180167+0000 mon.a (mon.0) 171 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T00:21:13.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: cluster 2024-09-13T00:21:13.187330+0000 mon.a (mon.0) 172 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:21:13.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: cluster 2024-09-13T00:21:13.207421+0000 mon.a (mon.0) 173 : cluster [DBG] fsmap 2024-09-13T00:21:13.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: cluster 2024-09-13T00:21:13.207502+0000 mon.a (mon.0) 174 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T00:21:13.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: cluster 2024-09-13T00:21:13.207819+0000 mon.a (mon.0) 175 : cluster [DBG] mgrmap e13: y(active, since 78s) 2024-09-13T00:21:13.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: cluster 2024-09-13T00:21:13.208134+0000 mon.a (mon.0) 176 : cluster [INF] overall HEALTH_OK 2024-09-13T00:21:13.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: audit 2024-09-13T00:21:13.216752+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: audit 2024-09-13T00:21:13.223233+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: audit 2024-09-13T00:21:13.230192+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: audit 2024-09-13T00:21:13.236057+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[22596]: audit 2024-09-13T00:21:13.241845+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.534 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:02.444446+0000 mon.a (mon.0) 140 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:21:13.534 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:02.444579+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:13.534 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:02.444741+0000 mon.a (mon.0) 142 : cluster [INF] mon.a calling monitor election 2024-09-13T00:21:13.534 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:03.169114+0000 mgr.y (mgr.14150) 50 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:13.534 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:03.439314+0000 mon.a (mon.0) 143 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:13.534 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:04.439876+0000 mon.a (mon.0) 144 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:13.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:04.441896+0000 mon.b (mon.1) 1 : cluster [INF] mon.b calling monitor election 2024-09-13T00:21:13.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:05.169438+0000 mgr.y (mgr.14150) 51 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:13.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:05.439896+0000 mon.a (mon.0) 145 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:13.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:06.132836+0000 mon.a (mon.0) 146 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:06.440016+0000 mon.a (mon.0) 147 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:13.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:07.133608+0000 mon.a (mon.0) 148 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:07.440174+0000 mon.a (mon.0) 149 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:13.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:07.471546+0000 mon.a (mon.0) 150 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T00:21:13.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:07.481978+0000 mon.a (mon.0) 151 : cluster [DBG] monmap e2: 2 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:21:13.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:07.499379+0000 mon.a (mon.0) 152 : cluster [DBG] fsmap 2024-09-13T00:21:13.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:07.499428+0000 mon.a (mon.0) 153 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T00:21:13.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:07.499680+0000 mon.a (mon.0) 154 : cluster [DBG] mgrmap e13: y(active, since 72s) 2024-09-13T00:21:13.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:07.499914+0000 mon.a (mon.0) 155 : cluster [INF] overall HEALTH_OK 2024-09-13T00:21:13.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:07.506401+0000 mon.a (mon.0) 156 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:07.513833+0000 mon.a (mon.0) 157 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:07.521251+0000 mon.a (mon.0) 158 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:07.528163+0000 mon.a (mon.0) 159 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:08.140862+0000 mon.a (mon.0) 162 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:21:13.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:08.141358+0000 mon.a (mon.0) 163 : cluster [INF] mon.a calling monitor election 2024-09-13T00:21:13.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:08.142314+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-13T00:21:13.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:08.159959+0000 mon.a (mon.0) 164 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:13.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:08.160291+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:09.134372+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.537 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:09.170144+0000 mgr.y (mgr.14150) 53 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:13.537 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:10.134536+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.537 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:10.136582+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-13T00:21:13.537 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:11.134683+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.537 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:11.170555+0000 mgr.y (mgr.14150) 54 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:13.537 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:12.134602+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.537 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:13.134773+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.537 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:13.180167+0000 mon.a (mon.0) 171 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T00:21:13.538 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:13.187330+0000 mon.a (mon.0) 172 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:21:13.538 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:13.207421+0000 mon.a (mon.0) 173 : cluster [DBG] fsmap 2024-09-13T00:21:13.538 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:13.207502+0000 mon.a (mon.0) 174 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T00:21:13.538 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:13.207819+0000 mon.a (mon.0) 175 : cluster [DBG] mgrmap e13: y(active, since 78s) 2024-09-13T00:21:13.538 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: cluster 2024-09-13T00:21:13.208134+0000 mon.a (mon.0) 176 : cluster [INF] overall HEALTH_OK 2024-09-13T00:21:13.538 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:13.216752+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.538 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:13.223233+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.538 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:13.230192+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.538 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:13.236057+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.538 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:13 smithi077 bash[32040]: audit 2024-09-13T00:21:13.241845+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: audit 2024-09-13T00:21:08.140862+0000 mon.a (mon.0) 162 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:21:13.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: cluster 2024-09-13T00:21:08.141358+0000 mon.a (mon.0) 163 : cluster [INF] mon.a calling monitor election 2024-09-13T00:21:13.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: cluster 2024-09-13T00:21:08.142314+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-13T00:21:13.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: audit 2024-09-13T00:21:08.159959+0000 mon.a (mon.0) 164 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:21:13.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: audit 2024-09-13T00:21:08.160291+0000 mon.a (mon.0) 165 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: audit 2024-09-13T00:21:09.134372+0000 mon.a (mon.0) 166 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: cluster 2024-09-13T00:21:09.170144+0000 mgr.y (mgr.14150) 53 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:13.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: audit 2024-09-13T00:21:10.134536+0000 mon.a (mon.0) 167 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: cluster 2024-09-13T00:21:10.136582+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-13T00:21:13.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: audit 2024-09-13T00:21:11.134683+0000 mon.a (mon.0) 168 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: cluster 2024-09-13T00:21:11.170555+0000 mgr.y (mgr.14150) 54 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:13.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: audit 2024-09-13T00:21:12.134602+0000 mon.a (mon.0) 169 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: audit 2024-09-13T00:21:13.134773+0000 mon.a (mon.0) 170 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:13.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: cluster 2024-09-13T00:21:13.180167+0000 mon.a (mon.0) 171 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T00:21:13.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: cluster 2024-09-13T00:21:13.187330+0000 mon.a (mon.0) 172 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:21:13.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: cluster 2024-09-13T00:21:13.207421+0000 mon.a (mon.0) 173 : cluster [DBG] fsmap 2024-09-13T00:21:13.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: cluster 2024-09-13T00:21:13.207502+0000 mon.a (mon.0) 174 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2024-09-13T00:21:13.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: cluster 2024-09-13T00:21:13.207819+0000 mon.a (mon.0) 175 : cluster [DBG] mgrmap e13: y(active, since 78s) 2024-09-13T00:21:13.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: cluster 2024-09-13T00:21:13.208134+0000 mon.a (mon.0) 176 : cluster [INF] overall HEALTH_OK 2024-09-13T00:21:13.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: audit 2024-09-13T00:21:13.216752+0000 mon.a (mon.0) 177 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: audit 2024-09-13T00:21:13.223233+0000 mon.a (mon.0) 178 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.607 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: audit 2024-09-13T00:21:13.230192+0000 mon.a (mon.0) 179 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.607 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: audit 2024-09-13T00:21:13.236057+0000 mon.a (mon.0) 180 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:13.607 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:13 smithi088 bash[24895]: audit 2024-09-13T00:21:13.241845+0000 mon.a (mon.0) 181 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:14.116 INFO:teuthology.orchestra.run.smithi088.stdout: 2024-09-13T00:21:14.116 INFO:teuthology.orchestra.run.smithi088.stdout:{"epoch":3,"fsid":"9218306c-7165-11ef-bceb-c7b262605968","modified":"2024-09-13T00:21:08.134606Z","created":"2024-09-13T00:18:54.475459Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:3300","nonce":0},{"type":"v1","addr":"172.21.15.77:6789","nonce":0}]},"addr":"172.21.15.77:6789/0","public_addr":"172.21.15.77:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:3300","nonce":0},{"type":"v1","addr":"172.21.15.88:6789","nonce":0}]},"addr":"172.21.15.88:6789/0","public_addr":"172.21.15.88:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:3301","nonce":0},{"type":"v1","addr":"172.21.15.77:6790","nonce":0}]},"addr":"172.21.15.77:6790/0","public_addr":"172.21.15.77:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-09-13T00:21:14.116 INFO:teuthology.orchestra.run.smithi088.stderr:dumped monmap epoch 3 2024-09-13T00:21:14.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: cluster 2024-09-13T00:21:13.170894+0000 mgr.y (mgr.14150) 55 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:14.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: audit 2024-09-13T00:21:13.628602+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:14.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: audit 2024-09-13T00:21:13.629906+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:14.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: cephadm 2024-09-13T00:21:13.631663+0000 mgr.y (mgr.14150) 56 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.conf 2024-09-13T00:21:14.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: cephadm 2024-09-13T00:21:13.632039+0000 mgr.y (mgr.14150) 57 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:21:14.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: cephadm 2024-09-13T00:21:13.750252+0000 mgr.y (mgr.14150) 58 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:21:14.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: cephadm 2024-09-13T00:21:13.764164+0000 mgr.y (mgr.14150) 59 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:21:14.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: audit 2024-09-13T00:21:13.870313+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:14.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: audit 2024-09-13T00:21:13.873915+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:14.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: audit 2024-09-13T00:21:13.895088+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:14.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: audit 2024-09-13T00:21:13.898033+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:14.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: audit 2024-09-13T00:21:13.901322+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:14.534 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: audit 2024-09-13T00:21:13.932058+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:21:14.534 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: audit 2024-09-13T00:21:13.933312+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:21:14.534 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: audit 2024-09-13T00:21:13.934335+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:14.534 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: audit 2024-09-13T00:21:14.115919+0000 mon.a (mon.0) 192 : audit [DBG] from='client.? 172.21.15.88:0/2964650821' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T00:21:14.534 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[22596]: audit 2024-09-13T00:21:14.133844+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:14.534 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: cluster 2024-09-13T00:21:13.170894+0000 mgr.y (mgr.14150) 55 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:14.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: audit 2024-09-13T00:21:13.628602+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:14.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: audit 2024-09-13T00:21:13.629906+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:14.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: cephadm 2024-09-13T00:21:13.631663+0000 mgr.y (mgr.14150) 56 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.conf 2024-09-13T00:21:14.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: cephadm 2024-09-13T00:21:13.632039+0000 mgr.y (mgr.14150) 57 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:21:14.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: cephadm 2024-09-13T00:21:13.750252+0000 mgr.y (mgr.14150) 58 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:21:14.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: cephadm 2024-09-13T00:21:13.764164+0000 mgr.y (mgr.14150) 59 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:21:14.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: audit 2024-09-13T00:21:13.870313+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:14.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: audit 2024-09-13T00:21:13.873915+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:14.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: audit 2024-09-13T00:21:13.895088+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:14.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: audit 2024-09-13T00:21:13.898033+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:14.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: audit 2024-09-13T00:21:13.901322+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:14.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: audit 2024-09-13T00:21:13.932058+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:21:14.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: audit 2024-09-13T00:21:13.933312+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:21:14.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: audit 2024-09-13T00:21:13.934335+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:14.537 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: audit 2024-09-13T00:21:14.115919+0000 mon.a (mon.0) 192 : audit [DBG] from='client.? 172.21.15.88:0/2964650821' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T00:21:14.537 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:14 smithi077 bash[32040]: audit 2024-09-13T00:21:14.133844+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:14.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: cluster 2024-09-13T00:21:13.170894+0000 mgr.y (mgr.14150) 55 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:14.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: audit 2024-09-13T00:21:13.628602+0000 mon.a (mon.0) 182 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:14.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: audit 2024-09-13T00:21:13.629906+0000 mon.a (mon.0) 183 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:14.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: cephadm 2024-09-13T00:21:13.631663+0000 mgr.y (mgr.14150) 56 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.conf 2024-09-13T00:21:14.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: cephadm 2024-09-13T00:21:13.632039+0000 mgr.y (mgr.14150) 57 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:21:14.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: cephadm 2024-09-13T00:21:13.750252+0000 mgr.y (mgr.14150) 58 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:21:14.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: cephadm 2024-09-13T00:21:13.764164+0000 mgr.y (mgr.14150) 59 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:21:14.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: audit 2024-09-13T00:21:13.870313+0000 mon.a (mon.0) 184 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:14.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: audit 2024-09-13T00:21:13.873915+0000 mon.a (mon.0) 185 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:14.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: audit 2024-09-13T00:21:13.895088+0000 mon.a (mon.0) 186 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:14.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: audit 2024-09-13T00:21:13.898033+0000 mon.a (mon.0) 187 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:14.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: audit 2024-09-13T00:21:13.901322+0000 mon.a (mon.0) 188 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:14.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: audit 2024-09-13T00:21:13.932058+0000 mon.a (mon.0) 189 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:21:14.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: audit 2024-09-13T00:21:13.933312+0000 mon.a (mon.0) 190 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:21:14.606 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: audit 2024-09-13T00:21:13.934335+0000 mon.a (mon.0) 191 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:14.607 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: audit 2024-09-13T00:21:14.115919+0000 mon.a (mon.0) 192 : audit [DBG] from='client.? 172.21.15.88:0/2964650821' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-13T00:21:14.607 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:14 smithi088 bash[24895]: audit 2024-09-13T00:21:14.133844+0000 mon.a (mon.0) 193 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:21:14.684 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-13T00:21:14.684 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-13T00:21:15.471 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:15 smithi077 bash[22596]: cephadm 2024-09-13T00:21:13.931569+0000 mgr.y (mgr.14150) 60 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2024-09-13T00:21:15.471 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:15 smithi077 bash[22596]: cephadm 2024-09-13T00:21:13.935295+0000 mgr.y (mgr.14150) 61 : cephadm [INF] Reconfiguring daemon mon.a on smithi077 2024-09-13T00:21:15.471 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:15 smithi077 bash[32040]: cephadm 2024-09-13T00:21:13.931569+0000 mgr.y (mgr.14150) 60 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2024-09-13T00:21:15.471 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:15 smithi077 bash[32040]: cephadm 2024-09-13T00:21:13.935295+0000 mgr.y (mgr.14150) 61 : cephadm [INF] Reconfiguring daemon mon.a on smithi077 2024-09-13T00:21:15.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:15 smithi088 bash[24895]: cephadm 2024-09-13T00:21:13.931569+0000 mgr.y (mgr.14150) 60 : cephadm [INF] Reconfiguring mon.a (unknown last config time)... 2024-09-13T00:21:15.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:15 smithi088 bash[24895]: cephadm 2024-09-13T00:21:13.935295+0000 mgr.y (mgr.14150) 61 : cephadm [INF] Reconfiguring daemon mon.a on smithi077 2024-09-13T00:21:16.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:16 smithi077 bash[32040]: cluster 2024-09-13T00:21:15.171164+0000 mgr.y (mgr.14150) 62 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:16.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:16 smithi077 bash[32040]: audit 2024-09-13T00:21:15.905125+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:16.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:16 smithi077 bash[32040]: audit 2024-09-13T00:21:15.911593+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:16.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:16 smithi077 bash[32040]: audit 2024-09-13T00:21:15.912657+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:21:16.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:16 smithi077 bash[32040]: audit 2024-09-13T00:21:15.913665+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:21:16.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:16 smithi077 bash[32040]: audit 2024-09-13T00:21:15.914476+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:16.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:16 smithi077 bash[22596]: cluster 2024-09-13T00:21:15.171164+0000 mgr.y (mgr.14150) 62 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:16.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:16 smithi077 bash[22596]: audit 2024-09-13T00:21:15.905125+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:16.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:16 smithi077 bash[22596]: audit 2024-09-13T00:21:15.911593+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:16.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:16 smithi077 bash[22596]: audit 2024-09-13T00:21:15.912657+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:21:16.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:16 smithi077 bash[22596]: audit 2024-09-13T00:21:15.913665+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:21:16.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:16 smithi077 bash[22596]: audit 2024-09-13T00:21:15.914476+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:16.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:16 smithi088 bash[24895]: cluster 2024-09-13T00:21:15.171164+0000 mgr.y (mgr.14150) 62 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:16.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:16 smithi088 bash[24895]: audit 2024-09-13T00:21:15.905125+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:16.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:16 smithi088 bash[24895]: audit 2024-09-13T00:21:15.911593+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:16.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:16 smithi088 bash[24895]: audit 2024-09-13T00:21:15.912657+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:21:16.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:16 smithi088 bash[24895]: audit 2024-09-13T00:21:15.913665+0000 mon.a (mon.0) 197 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:21:16.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:16 smithi088 bash[24895]: audit 2024-09-13T00:21:15.914476+0000 mon.a (mon.0) 198 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:17.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:17 smithi077 bash[22596]: cephadm 2024-09-13T00:21:15.912356+0000 mgr.y (mgr.14150) 63 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-13T00:21:17.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:17 smithi077 bash[22596]: cephadm 2024-09-13T00:21:15.915482+0000 mgr.y (mgr.14150) 64 : cephadm [INF] Reconfiguring daemon mon.c on smithi077 2024-09-13T00:21:17.462 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:17 smithi077 bash[32040]: cephadm 2024-09-13T00:21:15.912356+0000 mgr.y (mgr.14150) 63 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-13T00:21:17.462 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:17 smithi077 bash[32040]: cephadm 2024-09-13T00:21:15.915482+0000 mgr.y (mgr.14150) 64 : cephadm [INF] Reconfiguring daemon mon.c on smithi077 2024-09-13T00:21:17.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:17 smithi088 bash[24895]: cephadm 2024-09-13T00:21:15.912356+0000 mgr.y (mgr.14150) 63 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-13T00:21:17.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:17 smithi088 bash[24895]: cephadm 2024-09-13T00:21:15.915482+0000 mgr.y (mgr.14150) 64 : cephadm [INF] Reconfiguring daemon mon.c on smithi077 2024-09-13T00:21:18.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:18 smithi088 bash[24895]: cluster 2024-09-13T00:21:17.171495+0000 mgr.y (mgr.14150) 65 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:18.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:18 smithi088 bash[24895]: audit 2024-09-13T00:21:17.860762+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:18.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:18 smithi088 bash[24895]: audit 2024-09-13T00:21:17.867452+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:18.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:18 smithi088 bash[24895]: audit 2024-09-13T00:21:17.868908+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:21:18.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:18 smithi088 bash[24895]: audit 2024-09-13T00:21:17.870080+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:21:18.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:18 smithi088 bash[24895]: audit 2024-09-13T00:21:17.871051+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:18.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:18 smithi077 bash[22596]: cluster 2024-09-13T00:21:17.171495+0000 mgr.y (mgr.14150) 65 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:18.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:18 smithi077 bash[22596]: audit 2024-09-13T00:21:17.860762+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:18.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:18 smithi077 bash[22596]: audit 2024-09-13T00:21:17.867452+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:18.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:18 smithi077 bash[22596]: audit 2024-09-13T00:21:17.868908+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:21:18.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:18 smithi077 bash[22596]: audit 2024-09-13T00:21:17.870080+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:21:18.782 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:18 smithi077 bash[22596]: audit 2024-09-13T00:21:17.871051+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:18.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:18 smithi077 bash[32040]: cluster 2024-09-13T00:21:17.171495+0000 mgr.y (mgr.14150) 65 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:18.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:18 smithi077 bash[32040]: audit 2024-09-13T00:21:17.860762+0000 mon.a (mon.0) 199 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:18.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:18 smithi077 bash[32040]: audit 2024-09-13T00:21:17.867452+0000 mon.a (mon.0) 200 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:18.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:18 smithi077 bash[32040]: audit 2024-09-13T00:21:17.868908+0000 mon.a (mon.0) 201 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:21:18.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:18 smithi077 bash[32040]: audit 2024-09-13T00:21:17.870080+0000 mon.a (mon.0) 202 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:21:18.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:18 smithi077 bash[32040]: audit 2024-09-13T00:21:17.871051+0000 mon.a (mon.0) 203 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:19.402 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:21:19.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:19 smithi088 bash[24895]: cephadm 2024-09-13T00:21:17.868499+0000 mgr.y (mgr.14150) 66 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-13T00:21:19.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:19 smithi088 bash[24895]: cephadm 2024-09-13T00:21:17.872163+0000 mgr.y (mgr.14150) 67 : cephadm [INF] Reconfiguring daemon mon.b on smithi088 2024-09-13T00:21:19.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:19 smithi077 bash[22596]: cephadm 2024-09-13T00:21:17.868499+0000 mgr.y (mgr.14150) 66 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-13T00:21:19.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:19 smithi077 bash[22596]: cephadm 2024-09-13T00:21:17.872163+0000 mgr.y (mgr.14150) 67 : cephadm [INF] Reconfiguring daemon mon.b on smithi088 2024-09-13T00:21:19.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:19 smithi077 bash[32040]: cephadm 2024-09-13T00:21:17.868499+0000 mgr.y (mgr.14150) 66 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-13T00:21:19.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:19 smithi077 bash[32040]: cephadm 2024-09-13T00:21:17.872163+0000 mgr.y (mgr.14150) 67 : cephadm [INF] Reconfiguring daemon mon.b on smithi088 2024-09-13T00:21:20.746 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:20 smithi077 bash[32040]: cluster 2024-09-13T00:21:19.171907+0000 mgr.y (mgr.14150) 68 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:20.746 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:20 smithi077 bash[32040]: audit 2024-09-13T00:21:19.644800+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:20.747 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:20 smithi077 bash[32040]: audit 2024-09-13T00:21:19.654118+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:20.747 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:20 smithi077 bash[32040]: audit 2024-09-13T00:21:19.656245+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:20.747 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:20 smithi077 bash[32040]: audit 2024-09-13T00:21:19.659455+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:20.747 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:20 smithi077 bash[32040]: audit 2024-09-13T00:21:19.661370+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:20.747 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:20 smithi077 bash[32040]: audit 2024-09-13T00:21:19.672527+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:20.747 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:20 smithi077 bash[22596]: cluster 2024-09-13T00:21:19.171907+0000 mgr.y (mgr.14150) 68 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:20.747 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:20 smithi077 bash[22596]: audit 2024-09-13T00:21:19.644800+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:20.747 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:20 smithi077 bash[22596]: audit 2024-09-13T00:21:19.654118+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:20.747 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:20 smithi077 bash[22596]: audit 2024-09-13T00:21:19.656245+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:20.747 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:20 smithi077 bash[22596]: audit 2024-09-13T00:21:19.659455+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:20.748 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:20 smithi077 bash[22596]: audit 2024-09-13T00:21:19.661370+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:20.748 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:20 smithi077 bash[22596]: audit 2024-09-13T00:21:19.672527+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:21.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:20 smithi088 bash[24895]: cluster 2024-09-13T00:21:19.171907+0000 mgr.y (mgr.14150) 68 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:21.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:20 smithi088 bash[24895]: audit 2024-09-13T00:21:19.644800+0000 mon.a (mon.0) 204 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:21.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:20 smithi088 bash[24895]: audit 2024-09-13T00:21:19.654118+0000 mon.a (mon.0) 205 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:21.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:20 smithi088 bash[24895]: audit 2024-09-13T00:21:19.656245+0000 mon.a (mon.0) 206 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:21.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:20 smithi088 bash[24895]: audit 2024-09-13T00:21:19.659455+0000 mon.a (mon.0) 207 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:21.105 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:20 smithi088 bash[24895]: audit 2024-09-13T00:21:19.661370+0000 mon.a (mon.0) 208 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:21.105 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:20 smithi088 bash[24895]: audit 2024-09-13T00:21:19.672527+0000 mon.a (mon.0) 209 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:22.456 INFO:teuthology.orchestra.run.smithi077.stdout:# minimal ceph.conf for 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:21:22.457 INFO:teuthology.orchestra.run.smithi077.stdout:[global] 2024-09-13T00:21:22.457 INFO:teuthology.orchestra.run.smithi077.stdout: fsid = 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:21:22.457 INFO:teuthology.orchestra.run.smithi077.stdout: mon_host = [v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0] [v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0] [v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0] 2024-09-13T00:21:22.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:22 smithi077 bash[22596]: cluster 2024-09-13T00:21:21.172227+0000 mgr.y (mgr.14150) 69 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:22.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:22 smithi077 bash[22596]: audit 2024-09-13T00:21:22.456622+0000 mon.a (mon.0) 210 : audit [DBG] from='client.? 172.21.15.77:0/1950908062' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:22.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:22 smithi077 bash[32040]: cluster 2024-09-13T00:21:21.172227+0000 mgr.y (mgr.14150) 69 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:22.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:22 smithi077 bash[32040]: audit 2024-09-13T00:21:22.456622+0000 mon.a (mon.0) 210 : audit [DBG] from='client.? 172.21.15.77:0/1950908062' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:23.046 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-13T00:21:23.046 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:21:23.046 DEBUG:teuthology.orchestra.run.smithi077:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T00:21:23.060 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:21:23.060 DEBUG:teuthology.orchestra.run.smithi077:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:21:23.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:22 smithi088 bash[24895]: cluster 2024-09-13T00:21:21.172227+0000 mgr.y (mgr.14150) 69 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:23.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:22 smithi088 bash[24895]: audit 2024-09-13T00:21:22.456622+0000 mon.a (mon.0) 210 : audit [DBG] from='client.? 172.21.15.77:0/1950908062' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:23.113 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-13T00:21:23.113 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd of=/etc/ceph/ceph.conf 2024-09-13T00:21:23.129 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-13T00:21:23.129 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:21:23.186 INFO:tasks.cephadm:Adding mgr.y on smithi077 2024-09-13T00:21:23.186 INFO:tasks.cephadm:Adding mgr.x on smithi088 2024-09-13T00:21:23.186 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch apply mgr '2;smithi077=y;smithi088=x' 2024-09-13T00:21:25.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:24 smithi077 bash[22596]: cluster 2024-09-13T00:21:23.172535+0000 mgr.y (mgr.14150) 70 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:25.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:24 smithi077 bash[32040]: cluster 2024-09-13T00:21:23.172535+0000 mgr.y (mgr.14150) 70 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:25.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:24 smithi088 bash[24895]: cluster 2024-09-13T00:21:23.172535+0000 mgr.y (mgr.14150) 70 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:26.927 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T00:21:27.015 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:26 smithi088 bash[24895]: cluster 2024-09-13T00:21:25.172910+0000 mgr.y (mgr.14150) 71 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:27.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:26 smithi077 bash[22596]: cluster 2024-09-13T00:21:25.172910+0000 mgr.y (mgr.14150) 71 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:27.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:26 smithi077 bash[32040]: cluster 2024-09-13T00:21:25.172910+0000 mgr.y (mgr.14150) 71 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:28.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:28 smithi088 bash[24895]: cluster 2024-09-13T00:21:27.173306+0000 mgr.y (mgr.14150) 72 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:29.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:28 smithi077 bash[22596]: cluster 2024-09-13T00:21:27.173306+0000 mgr.y (mgr.14150) 72 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:29.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:28 smithi077 bash[32040]: cluster 2024-09-13T00:21:27.173306+0000 mgr.y (mgr.14150) 72 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:29.597 INFO:teuthology.orchestra.run.smithi088.stdout:Scheduled mgr update... 2024-09-13T00:21:30.326 DEBUG:teuthology.orchestra.run.smithi088:mgr.x> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@mgr.x.service 2024-09-13T00:21:30.328 INFO:tasks.cephadm:Deploying OSDs... 2024-09-13T00:21:30.328 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:21:30.328 DEBUG:teuthology.orchestra.run.smithi077:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T00:21:30.335 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T00:21:30.335 DEBUG:teuthology.orchestra.run.smithi077:> stat /dev/vg_nvme/lv_1 2024-09-13T00:21:30.384 INFO:teuthology.orchestra.run.smithi077.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T00:21:30.384 INFO:teuthology.orchestra.run.smithi077.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T00:21:30.384 INFO:teuthology.orchestra.run.smithi077.stdout:Device: 5h/5d Inode: 870 Links: 1 2024-09-13T00:21:30.385 INFO:teuthology.orchestra.run.smithi077.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T00:21:30.385 INFO:teuthology.orchestra.run.smithi077.stdout:Access: 2024-09-13 00:12:20.504167638 +0000 2024-09-13T00:21:30.385 INFO:teuthology.orchestra.run.smithi077.stdout:Modify: 2024-09-13 00:12:20.228170103 +0000 2024-09-13T00:21:30.385 INFO:teuthology.orchestra.run.smithi077.stdout:Change: 2024-09-13 00:12:20.228170103 +0000 2024-09-13T00:21:30.385 INFO:teuthology.orchestra.run.smithi077.stdout: Birth: - 2024-09-13T00:21:30.385 DEBUG:teuthology.orchestra.run.smithi077:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T00:21:30.437 INFO:teuthology.orchestra.run.smithi077.stderr:1+0 records in 2024-09-13T00:21:30.437 INFO:teuthology.orchestra.run.smithi077.stderr:1+0 records out 2024-09-13T00:21:30.437 INFO:teuthology.orchestra.run.smithi077.stderr:512 bytes copied, 0.000456587 s, 1.1 MB/s 2024-09-13T00:21:30.439 DEBUG:teuthology.orchestra.run.smithi077:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T00:21:30.492 DEBUG:teuthology.orchestra.run.smithi077:> stat /dev/vg_nvme/lv_2 2024-09-13T00:21:30.540 INFO:teuthology.orchestra.run.smithi077.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T00:21:30.541 INFO:teuthology.orchestra.run.smithi077.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T00:21:30.541 INFO:teuthology.orchestra.run.smithi077.stdout:Device: 5h/5d Inode: 890 Links: 1 2024-09-13T00:21:30.541 INFO:teuthology.orchestra.run.smithi077.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T00:21:30.541 INFO:teuthology.orchestra.run.smithi077.stdout:Access: 2024-09-13 00:12:20.756165387 +0000 2024-09-13T00:21:30.541 INFO:teuthology.orchestra.run.smithi077.stdout:Modify: 2024-09-13 00:12:20.752165423 +0000 2024-09-13T00:21:30.541 INFO:teuthology.orchestra.run.smithi077.stdout:Change: 2024-09-13 00:12:20.752165423 +0000 2024-09-13T00:21:30.541 INFO:teuthology.orchestra.run.smithi077.stdout: Birth: - 2024-09-13T00:21:30.541 DEBUG:teuthology.orchestra.run.smithi077:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T00:21:30.593 INFO:teuthology.orchestra.run.smithi077.stderr:1+0 records in 2024-09-13T00:21:30.593 INFO:teuthology.orchestra.run.smithi077.stderr:1+0 records out 2024-09-13T00:21:30.593 INFO:teuthology.orchestra.run.smithi077.stderr:512 bytes copied, 0.000357883 s, 1.4 MB/s 2024-09-13T00:21:30.595 DEBUG:teuthology.orchestra.run.smithi077:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T00:21:30.645 DEBUG:teuthology.orchestra.run.smithi077:> stat /dev/vg_nvme/lv_3 2024-09-13T00:21:30.690 INFO:teuthology.orchestra.run.smithi077.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T00:21:30.690 INFO:teuthology.orchestra.run.smithi077.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T00:21:30.690 INFO:teuthology.orchestra.run.smithi077.stdout:Device: 5h/5d Inode: 907 Links: 1 2024-09-13T00:21:30.691 INFO:teuthology.orchestra.run.smithi077.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T00:21:30.691 INFO:teuthology.orchestra.run.smithi077.stdout:Access: 2024-09-13 00:12:21.228161170 +0000 2024-09-13T00:21:30.691 INFO:teuthology.orchestra.run.smithi077.stdout:Modify: 2024-09-13 00:12:21.224161207 +0000 2024-09-13T00:21:30.691 INFO:teuthology.orchestra.run.smithi077.stdout:Change: 2024-09-13 00:12:21.224161207 +0000 2024-09-13T00:21:30.691 INFO:teuthology.orchestra.run.smithi077.stdout: Birth: - 2024-09-13T00:21:30.691 DEBUG:teuthology.orchestra.run.smithi077:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T00:21:30.743 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[32040]: cluster 2024-09-13T00:21:29.173702+0000 mgr.y (mgr.14150) 73 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:30.743 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[32040]: audit 2024-09-13T00:21:29.586743+0000 mgr.y (mgr.14150) 74 : audit [DBG] from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi077=y;smithi088=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:21:30.743 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[32040]: cephadm 2024-09-13T00:21:29.589631+0000 mgr.y (mgr.14150) 75 : cephadm [INF] Saving service mgr spec with placement smithi077=y;smithi088=x;count:2 2024-09-13T00:21:30.743 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[32040]: audit 2024-09-13T00:21:29.596072+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:30.743 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[32040]: audit 2024-09-13T00:21:29.597696+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:30.744 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[32040]: audit 2024-09-13T00:21:29.601174+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:30.744 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[32040]: audit 2024-09-13T00:21:29.602957+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:30.744 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[32040]: audit 2024-09-13T00:21:29.612153+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:30.744 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[32040]: audit 2024-09-13T00:21:29.615103+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:21:30.744 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[32040]: audit 2024-09-13T00:21:29.620772+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T00:21:30.744 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[32040]: audit 2024-09-13T00:21:29.622453+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:21:30.744 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[32040]: audit 2024-09-13T00:21:29.624279+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:30.744 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[32040]: cephadm 2024-09-13T00:21:29.626176+0000 mgr.y (mgr.14150) 76 : cephadm [INF] Deploying daemon mgr.x on smithi088 2024-09-13T00:21:30.744 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[22596]: cluster 2024-09-13T00:21:29.173702+0000 mgr.y (mgr.14150) 73 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:30.744 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[22596]: audit 2024-09-13T00:21:29.586743+0000 mgr.y (mgr.14150) 74 : audit [DBG] from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi077=y;smithi088=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:21:30.744 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[22596]: cephadm 2024-09-13T00:21:29.589631+0000 mgr.y (mgr.14150) 75 : cephadm [INF] Saving service mgr spec with placement smithi077=y;smithi088=x;count:2 2024-09-13T00:21:30.745 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[22596]: audit 2024-09-13T00:21:29.596072+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:30.745 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[22596]: audit 2024-09-13T00:21:29.597696+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:30.745 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[22596]: audit 2024-09-13T00:21:29.601174+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:30.745 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[22596]: audit 2024-09-13T00:21:29.602957+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:30.745 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[22596]: audit 2024-09-13T00:21:29.612153+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:30.745 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[22596]: audit 2024-09-13T00:21:29.615103+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:21:30.745 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[22596]: audit 2024-09-13T00:21:29.620772+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T00:21:30.745 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[22596]: audit 2024-09-13T00:21:29.622453+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:21:30.745 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[22596]: audit 2024-09-13T00:21:29.624279+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:30.745 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:30 smithi077 bash[22596]: cephadm 2024-09-13T00:21:29.626176+0000 mgr.y (mgr.14150) 76 : cephadm [INF] Deploying daemon mgr.x on smithi088 2024-09-13T00:21:30.746 INFO:teuthology.orchestra.run.smithi077.stderr:1+0 records in 2024-09-13T00:21:30.746 INFO:teuthology.orchestra.run.smithi077.stderr:1+0 records out 2024-09-13T00:21:30.746 INFO:teuthology.orchestra.run.smithi077.stderr:512 bytes copied, 0.000431626 s, 1.2 MB/s 2024-09-13T00:21:30.747 DEBUG:teuthology.orchestra.run.smithi077:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T00:21:30.796 DEBUG:teuthology.orchestra.run.smithi077:> stat /dev/vg_nvme/lv_4 2024-09-13T00:21:30.842 INFO:teuthology.orchestra.run.smithi077.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T00:21:30.842 INFO:teuthology.orchestra.run.smithi077.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T00:21:30.842 INFO:teuthology.orchestra.run.smithi077.stdout:Device: 5h/5d Inode: 923 Links: 1 2024-09-13T00:21:30.842 INFO:teuthology.orchestra.run.smithi077.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T00:21:30.842 INFO:teuthology.orchestra.run.smithi077.stdout:Access: 2024-09-13 00:12:21.940154811 +0000 2024-09-13T00:21:30.842 INFO:teuthology.orchestra.run.smithi077.stdout:Modify: 2024-09-13 00:12:21.704156919 +0000 2024-09-13T00:21:30.842 INFO:teuthology.orchestra.run.smithi077.stdout:Change: 2024-09-13 00:12:21.704156919 +0000 2024-09-13T00:21:30.843 INFO:teuthology.orchestra.run.smithi077.stdout: Birth: - 2024-09-13T00:21:30.843 DEBUG:teuthology.orchestra.run.smithi077:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T00:21:30.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:30 smithi088 bash[24895]: cluster 2024-09-13T00:21:29.173702+0000 mgr.y (mgr.14150) 73 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:30.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:30 smithi088 bash[24895]: audit 2024-09-13T00:21:29.586743+0000 mgr.y (mgr.14150) 74 : audit [DBG] from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi077=y;smithi088=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:21:30.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:30 smithi088 bash[24895]: cephadm 2024-09-13T00:21:29.589631+0000 mgr.y (mgr.14150) 75 : cephadm [INF] Saving service mgr spec with placement smithi077=y;smithi088=x;count:2 2024-09-13T00:21:30.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:30 smithi088 bash[24895]: audit 2024-09-13T00:21:29.596072+0000 mon.a (mon.0) 211 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:30.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:30 smithi088 bash[24895]: audit 2024-09-13T00:21:29.597696+0000 mon.a (mon.0) 212 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:30.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:30 smithi088 bash[24895]: audit 2024-09-13T00:21:29.601174+0000 mon.a (mon.0) 213 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:30.855 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:30 smithi088 bash[24895]: audit 2024-09-13T00:21:29.602957+0000 mon.a (mon.0) 214 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:30.855 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:30 smithi088 bash[24895]: audit 2024-09-13T00:21:29.612153+0000 mon.a (mon.0) 215 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:30.855 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:30 smithi088 bash[24895]: audit 2024-09-13T00:21:29.615103+0000 mon.a (mon.0) 216 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:21:30.855 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:30 smithi088 bash[24895]: audit 2024-09-13T00:21:29.620772+0000 mon.a (mon.0) 217 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-13T00:21:30.855 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:30 smithi088 bash[24895]: audit 2024-09-13T00:21:29.622453+0000 mon.a (mon.0) 218 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:21:30.855 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:30 smithi088 bash[24895]: audit 2024-09-13T00:21:29.624279+0000 mon.a (mon.0) 219 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:30.855 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:30 smithi088 bash[24895]: cephadm 2024-09-13T00:21:29.626176+0000 mgr.y (mgr.14150) 76 : cephadm [INF] Deploying daemon mgr.x on smithi088 2024-09-13T00:21:30.897 INFO:teuthology.orchestra.run.smithi077.stderr:1+0 records in 2024-09-13T00:21:30.897 INFO:teuthology.orchestra.run.smithi077.stderr:1+0 records out 2024-09-13T00:21:30.897 INFO:teuthology.orchestra.run.smithi077.stderr:512 bytes copied, 0.000369572 s, 1.4 MB/s 2024-09-13T00:21:30.899 DEBUG:teuthology.orchestra.run.smithi077:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T00:21:30.948 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-13T00:21:30.948 DEBUG:teuthology.orchestra.run.smithi088:> dd if=/scratch_devs of=/dev/stdout 2024-09-13T00:21:30.955 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-13T00:21:30.955 DEBUG:teuthology.orchestra.run.smithi088:> stat /dev/vg_nvme/lv_1 2024-09-13T00:21:31.002 INFO:teuthology.orchestra.run.smithi088.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-13T00:21:31.002 INFO:teuthology.orchestra.run.smithi088.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T00:21:31.002 INFO:teuthology.orchestra.run.smithi088.stdout:Device: 5h/5d Inode: 876 Links: 1 2024-09-13T00:21:31.002 INFO:teuthology.orchestra.run.smithi088.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T00:21:31.003 INFO:teuthology.orchestra.run.smithi088.stdout:Access: 2024-09-13 00:12:17.465900978 +0000 2024-09-13T00:21:31.003 INFO:teuthology.orchestra.run.smithi088.stdout:Modify: 2024-09-13 00:12:17.241902979 +0000 2024-09-13T00:21:31.003 INFO:teuthology.orchestra.run.smithi088.stdout:Change: 2024-09-13 00:12:17.241902979 +0000 2024-09-13T00:21:31.003 INFO:teuthology.orchestra.run.smithi088.stdout: Birth: - 2024-09-13T00:21:31.004 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-13T00:21:31.054 INFO:teuthology.orchestra.run.smithi088.stderr:1+0 records in 2024-09-13T00:21:31.054 INFO:teuthology.orchestra.run.smithi088.stderr:1+0 records out 2024-09-13T00:21:31.054 INFO:teuthology.orchestra.run.smithi088.stderr:512 bytes copied, 0.000279404 s, 1.8 MB/s 2024-09-13T00:21:31.055 DEBUG:teuthology.orchestra.run.smithi088:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-13T00:21:31.101 DEBUG:teuthology.orchestra.run.smithi088:> stat /dev/vg_nvme/lv_2 2024-09-13T00:21:31.150 INFO:teuthology.orchestra.run.smithi088.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-13T00:21:31.150 INFO:teuthology.orchestra.run.smithi088.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T00:21:31.150 INFO:teuthology.orchestra.run.smithi088.stdout:Device: 5h/5d Inode: 886 Links: 1 2024-09-13T00:21:31.150 INFO:teuthology.orchestra.run.smithi088.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T00:21:31.150 INFO:teuthology.orchestra.run.smithi088.stdout:Access: 2024-09-13 00:12:17.925896869 +0000 2024-09-13T00:21:31.150 INFO:teuthology.orchestra.run.smithi088.stdout:Modify: 2024-09-13 00:12:17.693898941 +0000 2024-09-13T00:21:31.150 INFO:teuthology.orchestra.run.smithi088.stdout:Change: 2024-09-13 00:12:17.693898941 +0000 2024-09-13T00:21:31.150 INFO:teuthology.orchestra.run.smithi088.stdout: Birth: - 2024-09-13T00:21:31.151 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-13T00:21:31.203 INFO:teuthology.orchestra.run.smithi088.stderr:1+0 records in 2024-09-13T00:21:31.203 INFO:teuthology.orchestra.run.smithi088.stderr:1+0 records out 2024-09-13T00:21:31.203 INFO:teuthology.orchestra.run.smithi088.stderr:512 bytes copied, 0.000306624 s, 1.7 MB/s 2024-09-13T00:21:31.204 DEBUG:teuthology.orchestra.run.smithi088:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-13T00:21:31.251 DEBUG:teuthology.orchestra.run.smithi088:> stat /dev/vg_nvme/lv_3 2024-09-13T00:21:31.297 INFO:teuthology.orchestra.run.smithi088.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-13T00:21:31.297 INFO:teuthology.orchestra.run.smithi088.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T00:21:31.297 INFO:teuthology.orchestra.run.smithi088.stdout:Device: 5h/5d Inode: 911 Links: 1 2024-09-13T00:21:31.297 INFO:teuthology.orchestra.run.smithi088.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T00:21:31.297 INFO:teuthology.orchestra.run.smithi088.stdout:Access: 2024-09-13 00:12:18.401892617 +0000 2024-09-13T00:21:31.297 INFO:teuthology.orchestra.run.smithi088.stdout:Modify: 2024-09-13 00:12:18.161894761 +0000 2024-09-13T00:21:31.297 INFO:teuthology.orchestra.run.smithi088.stdout:Change: 2024-09-13 00:12:18.161894761 +0000 2024-09-13T00:21:31.297 INFO:teuthology.orchestra.run.smithi088.stdout: Birth: - 2024-09-13T00:21:31.297 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-13T00:21:31.350 INFO:teuthology.orchestra.run.smithi088.stderr:1+0 records in 2024-09-13T00:21:31.350 INFO:teuthology.orchestra.run.smithi088.stderr:1+0 records out 2024-09-13T00:21:31.350 INFO:teuthology.orchestra.run.smithi088.stderr:512 bytes copied, 0.000306863 s, 1.7 MB/s 2024-09-13T00:21:31.351 DEBUG:teuthology.orchestra.run.smithi088:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-13T00:21:31.398 DEBUG:teuthology.orchestra.run.smithi088:> stat /dev/vg_nvme/lv_4 2024-09-13T00:21:31.445 INFO:teuthology.orchestra.run.smithi088.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-13T00:21:31.446 INFO:teuthology.orchestra.run.smithi088.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-13T00:21:31.446 INFO:teuthology.orchestra.run.smithi088.stdout:Device: 5h/5d Inode: 923 Links: 1 2024-09-13T00:21:31.446 INFO:teuthology.orchestra.run.smithi088.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-13T00:21:31.446 INFO:teuthology.orchestra.run.smithi088.stdout:Access: 2024-09-13 00:12:18.913888044 +0000 2024-09-13T00:21:31.446 INFO:teuthology.orchestra.run.smithi088.stdout:Modify: 2024-09-13 00:12:18.641890472 +0000 2024-09-13T00:21:31.446 INFO:teuthology.orchestra.run.smithi088.stdout:Change: 2024-09-13 00:12:18.641890472 +0000 2024-09-13T00:21:31.446 INFO:teuthology.orchestra.run.smithi088.stdout: Birth: - 2024-09-13T00:21:31.446 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-13T00:21:31.500 INFO:teuthology.orchestra.run.smithi088.stderr:1+0 records in 2024-09-13T00:21:31.500 INFO:teuthology.orchestra.run.smithi088.stderr:1+0 records out 2024-09-13T00:21:31.500 INFO:teuthology.orchestra.run.smithi088.stderr:512 bytes copied, 0.000337894 s, 1.5 MB/s 2024-09-13T00:21:31.501 DEBUG:teuthology.orchestra.run.smithi088:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-13T00:21:31.551 INFO:tasks.cephadm:Deploying osd.0 on smithi077 with /dev/vg_nvme/lv_4... 2024-09-13T00:21:31.551 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-09-13T00:21:31.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:31 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:21:31.854 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:31 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:21:31.854 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:31 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:21:31.854 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:31 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:21:32.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:31 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:21:32.354 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:31 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:21:32.354 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:32 smithi088 systemd[1]: Started Ceph mgr.x for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:21:32.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:32 smithi088 bash[24895]: cluster 2024-09-13T00:21:31.174134+0000 mgr.y (mgr.14150) 77 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:32.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:32 smithi088 bash[24895]: audit 2024-09-13T00:21:32.093834+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:32.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:32 smithi088 bash[24895]: audit 2024-09-13T00:21:32.103304+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:32.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:32 smithi088 bash[24895]: audit 2024-09-13T00:21:32.112996+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:32.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:32 smithi088 bash[24895]: audit 2024-09-13T00:21:32.122216+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:32.855 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:32 smithi088 bash[24895]: audit 2024-09-13T00:21:32.178771+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:33.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:32 smithi077 bash[22596]: cluster 2024-09-13T00:21:31.174134+0000 mgr.y (mgr.14150) 77 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:33.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:32 smithi077 bash[22596]: audit 2024-09-13T00:21:32.093834+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:33.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:32 smithi077 bash[22596]: audit 2024-09-13T00:21:32.103304+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:33.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:32 smithi077 bash[22596]: audit 2024-09-13T00:21:32.112996+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:33.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:32 smithi077 bash[22596]: audit 2024-09-13T00:21:32.122216+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:33.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:32 smithi077 bash[22596]: audit 2024-09-13T00:21:32.178771+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:33.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:32 smithi077 bash[32040]: cluster 2024-09-13T00:21:31.174134+0000 mgr.y (mgr.14150) 77 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:33.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:32 smithi077 bash[32040]: audit 2024-09-13T00:21:32.093834+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:33.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:32 smithi077 bash[32040]: audit 2024-09-13T00:21:32.103304+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:33.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:32 smithi077 bash[32040]: audit 2024-09-13T00:21:32.112996+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:33.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:32 smithi077 bash[32040]: audit 2024-09-13T00:21:32.122216+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:33.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:32 smithi077 bash[32040]: audit 2024-09-13T00:21:32.178771+0000 mon.a (mon.0) 224 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:33.354 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:33 smithi088 bash[25838]: debug 2024-09-13T00:21:33.034+0000 7f257ced6640 1 -- 172.21.15.88:0/96340440 <== mon.1 v2:172.21.15.88:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5654d48b6340 con 0x5654d48c2800 2024-09-13T00:21:33.354 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:33 smithi088 bash[25838]: debug 2024-09-13T00:21:33.142+0000 7f257f73eec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T00:21:34.518 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:34 smithi088 bash[25838]: debug 2024-09-13T00:21:34.302+0000 7f257f73eec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T00:21:34.519 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:34 smithi088 bash[25838]: debug 2024-09-13T00:21:34.366+0000 7f257f73eec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T00:21:34.519 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:34 smithi088 bash[24895]: cluster 2024-09-13T00:21:33.174619+0000 mgr.y (mgr.14150) 78 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:34.519 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:34 smithi088 bash[24895]: audit 2024-09-13T00:21:33.224515+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:34.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:34 smithi077 bash[22596]: cluster 2024-09-13T00:21:33.174619+0000 mgr.y (mgr.14150) 78 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:34.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:34 smithi077 bash[22596]: audit 2024-09-13T00:21:33.224515+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:34.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:34 smithi077 bash[32040]: cluster 2024-09-13T00:21:33.174619+0000 mgr.y (mgr.14150) 78 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:34.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:34 smithi077 bash[32040]: audit 2024-09-13T00:21:33.224515+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:34.854 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:34 smithi088 bash[25838]: debug 2024-09-13T00:21:34.514+0000 7f257f73eec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T00:21:35.426 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:35 smithi088 bash[25838]: debug 2024-09-13T00:21:35.134+0000 7f257f73eec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T00:21:35.426 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:35 smithi088 bash[25838]: debug 2024-09-13T00:21:35.202+0000 7f257f73eec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T00:21:35.426 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:35 smithi088 bash[25838]: debug 2024-09-13T00:21:35.266+0000 7f257f73eec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T00:21:35.788 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:35 smithi088 bash[25838]: debug 2024-09-13T00:21:35.422+0000 7f257f73eec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T00:21:35.788 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:35 smithi088 bash[25838]: debug 2024-09-13T00:21:35.494+0000 7f257f73eec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T00:21:35.788 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:35 smithi088 bash[25838]: debug 2024-09-13T00:21:35.558+0000 7f257f73eec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T00:21:36.088 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:35 smithi088 bash[25838]: debug 2024-09-13T00:21:35.782+0000 7f257f73eec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T00:21:36.088 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:35 smithi088 bash[25838]: debug 2024-09-13T00:21:35.846+0000 7f257f73eec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T00:21:36.088 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:36 smithi088 bash[25838]: debug 2024-09-13T00:21:36.010+0000 7f257f73eec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T00:21:36.276 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:21:36.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:36 smithi088 bash[24895]: cluster 2024-09-13T00:21:35.175023+0000 mgr.y (mgr.14150) 79 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:36.354 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:36 smithi088 bash[25838]: debug 2024-09-13T00:21:36.082+0000 7f257f73eec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T00:21:36.354 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:36 smithi088 bash[25838]: debug 2024-09-13T00:21:36.310+0000 7f257f73eec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:21:36.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:36 smithi077 bash[22596]: cluster 2024-09-13T00:21:35.175023+0000 mgr.y (mgr.14150) 79 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:36.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:36 smithi077 bash[32040]: cluster 2024-09-13T00:21:35.175023+0000 mgr.y (mgr.14150) 79 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:36.854 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:36 smithi088 bash[25838]: debug 2024-09-13T00:21:36.470+0000 7f257f73eec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T00:21:36.854 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:36 smithi088 bash[25838]: debug 2024-09-13T00:21:36.546+0000 7f257f73eec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T00:21:36.854 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:36 smithi088 bash[25838]: debug 2024-09-13T00:21:36.694+0000 7f257f73eec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:21:37.724 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:37 smithi088 bash[25838]: debug 2024-09-13T00:21:37.454+0000 7f257f73eec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T00:21:37.724 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:37 smithi088 bash[25838]: debug 2024-09-13T00:21:37.526+0000 7f257f73eec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T00:21:38.104 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:37 smithi088 bash[25838]: debug 2024-09-13T00:21:37.718+0000 7f257f73eec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T00:21:38.104 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:37 smithi088 bash[25838]: debug 2024-09-13T00:21:37.790+0000 7f257f73eec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T00:21:38.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[22596]: audit 2024-09-13T00:21:37.122732+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:38.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[22596]: audit 2024-09-13T00:21:37.129998+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:38.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[22596]: audit 2024-09-13T00:21:37.131577+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:38.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[22596]: audit 2024-09-13T00:21:37.133599+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:38.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[22596]: audit 2024-09-13T00:21:37.142857+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:38.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[22596]: cluster 2024-09-13T00:21:37.175361+0000 mgr.y (mgr.14150) 80 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:38.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[22596]: cephadm 2024-09-13T00:21:37.201661+0000 mgr.y (mgr.14150) 81 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-09-13T00:21:38.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[22596]: audit 2024-09-13T00:21:37.202307+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:21:38.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[22596]: audit 2024-09-13T00:21:37.203832+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:21:38.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[22596]: audit 2024-09-13T00:21:37.205304+0000 mon.a (mon.0) 233 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:38.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[22596]: cephadm 2024-09-13T00:21:37.206923+0000 mgr.y (mgr.14150) 82 : cephadm [INF] Reconfiguring daemon mgr.y on smithi077 2024-09-13T00:21:38.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[32040]: audit 2024-09-13T00:21:37.122732+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:38.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[32040]: audit 2024-09-13T00:21:37.129998+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:38.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[32040]: audit 2024-09-13T00:21:37.131577+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:38.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[32040]: audit 2024-09-13T00:21:37.133599+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:38.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[32040]: audit 2024-09-13T00:21:37.142857+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:38.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[32040]: cluster 2024-09-13T00:21:37.175361+0000 mgr.y (mgr.14150) 80 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:38.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[32040]: cephadm 2024-09-13T00:21:37.201661+0000 mgr.y (mgr.14150) 81 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-09-13T00:21:38.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[32040]: audit 2024-09-13T00:21:37.202307+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:21:38.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[32040]: audit 2024-09-13T00:21:37.203832+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:21:38.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[32040]: audit 2024-09-13T00:21:37.205304+0000 mon.a (mon.0) 233 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:38.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:38 smithi077 bash[32040]: cephadm 2024-09-13T00:21:37.206923+0000 mgr.y (mgr.14150) 82 : cephadm [INF] Reconfiguring daemon mgr.y on smithi077 2024-09-13T00:21:38.604 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:38 smithi088 bash[25838]: debug 2024-09-13T00:21:38.098+0000 7f257f73eec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T00:21:38.604 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:21:38 smithi088 bash[25838]: debug 2024-09-13T00:21:38.174+0000 7f257f73eec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T00:21:38.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:38 smithi088 bash[24895]: audit 2024-09-13T00:21:37.122732+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:38.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:38 smithi088 bash[24895]: audit 2024-09-13T00:21:37.129998+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:38.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:38 smithi088 bash[24895]: audit 2024-09-13T00:21:37.131577+0000 mon.a (mon.0) 228 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:38.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:38 smithi088 bash[24895]: audit 2024-09-13T00:21:37.133599+0000 mon.a (mon.0) 229 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:38.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:38 smithi088 bash[24895]: audit 2024-09-13T00:21:37.142857+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:38.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:38 smithi088 bash[24895]: cluster 2024-09-13T00:21:37.175361+0000 mgr.y (mgr.14150) 80 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:38.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:38 smithi088 bash[24895]: cephadm 2024-09-13T00:21:37.201661+0000 mgr.y (mgr.14150) 81 : cephadm [INF] Reconfiguring mgr.y (unknown last config time)... 2024-09-13T00:21:38.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:38 smithi088 bash[24895]: audit 2024-09-13T00:21:37.202307+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:21:38.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:38 smithi088 bash[24895]: audit 2024-09-13T00:21:37.203832+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:21:38.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:38 smithi088 bash[24895]: audit 2024-09-13T00:21:37.205304+0000 mon.a (mon.0) 233 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:38.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:38 smithi088 bash[24895]: cephadm 2024-09-13T00:21:37.206923+0000 mgr.y (mgr.14150) 82 : cephadm [INF] Reconfiguring daemon mgr.y on smithi077 2024-09-13T00:21:39.412 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:39 smithi077 bash[32040]: cluster 2024-09-13T00:21:38.205258+0000 mon.a (mon.0) 234 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:21:39.412 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:39 smithi077 bash[32040]: audit 2024-09-13T00:21:38.206668+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.88:0/3544546420' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:21:39.412 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:39 smithi077 bash[32040]: audit 2024-09-13T00:21:38.207201+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.88:0/3544546420' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:21:39.413 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:39 smithi077 bash[32040]: audit 2024-09-13T00:21:38.208348+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.88:0/3544546420' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:21:39.413 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:39 smithi077 bash[32040]: audit 2024-09-13T00:21:38.208895+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.88:0/3544546420' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:21:39.413 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:39 smithi077 bash[22596]: cluster 2024-09-13T00:21:38.205258+0000 mon.a (mon.0) 234 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:21:39.413 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:39 smithi077 bash[22596]: audit 2024-09-13T00:21:38.206668+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.88:0/3544546420' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:21:39.413 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:39 smithi077 bash[22596]: audit 2024-09-13T00:21:38.207201+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.88:0/3544546420' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:21:39.413 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:39 smithi077 bash[22596]: audit 2024-09-13T00:21:38.208348+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.88:0/3544546420' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:21:39.413 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:39 smithi077 bash[22596]: audit 2024-09-13T00:21:38.208895+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.88:0/3544546420' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:21:39.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:39 smithi088 bash[24895]: cluster 2024-09-13T00:21:38.205258+0000 mon.a (mon.0) 234 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:21:39.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:39 smithi088 bash[24895]: audit 2024-09-13T00:21:38.206668+0000 mon.b (mon.1) 3 : audit [DBG] from='mgr.? 172.21.15.88:0/3544546420' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:21:39.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:39 smithi088 bash[24895]: audit 2024-09-13T00:21:38.207201+0000 mon.b (mon.1) 4 : audit [DBG] from='mgr.? 172.21.15.88:0/3544546420' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:21:39.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:39 smithi088 bash[24895]: audit 2024-09-13T00:21:38.208348+0000 mon.b (mon.1) 5 : audit [DBG] from='mgr.? 172.21.15.88:0/3544546420' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:21:39.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:39 smithi088 bash[24895]: audit 2024-09-13T00:21:38.208895+0000 mon.b (mon.1) 6 : audit [DBG] from='mgr.? 172.21.15.88:0/3544546420' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:21:40.087 INFO:teuthology.orchestra.run.smithi077.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-09-13T00:21:40.278 INFO:teuthology.orchestra.run.smithi077.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-09-13T00:21:40.296 INFO:teuthology.orchestra.run.smithi077.stderr: stderr: 10+0 records in 2024-09-13T00:21:40.296 INFO:teuthology.orchestra.run.smithi077.stderr:10+0 records out 2024-09-13T00:21:40.296 INFO:teuthology.orchestra.run.smithi077.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0155059 s, 676 MB/s 2024-09-13T00:21:40.297 INFO:teuthology.orchestra.run.smithi077.stderr:--> Zapping successful for: 2024-09-13T00:21:40.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:40 smithi077 bash[22596]: cluster 2024-09-13T00:21:39.164026+0000 mon.a (mon.0) 235 : cluster [DBG] mgrmap e14: y(active, since 104s), standbys: x 2024-09-13T00:21:40.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:40 smithi077 bash[22596]: audit 2024-09-13T00:21:39.164200+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T00:21:40.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:40 smithi077 bash[22596]: cluster 2024-09-13T00:21:39.175591+0000 mgr.y (mgr.14150) 83 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:40.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:40 smithi077 bash[32040]: cluster 2024-09-13T00:21:39.164026+0000 mon.a (mon.0) 235 : cluster [DBG] mgrmap e14: y(active, since 104s), standbys: x 2024-09-13T00:21:40.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:40 smithi077 bash[32040]: audit 2024-09-13T00:21:39.164200+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T00:21:40.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:40 smithi077 bash[32040]: cluster 2024-09-13T00:21:39.175591+0000 mgr.y (mgr.14150) 83 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:40.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:40 smithi088 bash[24895]: cluster 2024-09-13T00:21:39.164026+0000 mon.a (mon.0) 235 : cluster [DBG] mgrmap e14: y(active, since 104s), standbys: x 2024-09-13T00:21:40.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:40 smithi088 bash[24895]: audit 2024-09-13T00:21:39.164200+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T00:21:40.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:40 smithi088 bash[24895]: cluster 2024-09-13T00:21:39.175591+0000 mgr.y (mgr.14150) 83 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:40.948 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi077:vg_nvme/lv_4 2024-09-13T00:21:41.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:41 smithi088 bash[24895]: audit 2024-09-13T00:21:40.546290+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:41.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:41 smithi088 bash[24895]: audit 2024-09-13T00:21:40.550980+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:41.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:41 smithi088 bash[24895]: audit 2024-09-13T00:21:40.552587+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:41.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:41 smithi088 bash[24895]: audit 2024-09-13T00:21:40.979834+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:41.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:41 smithi088 bash[24895]: audit 2024-09-13T00:21:40.981021+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:41.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:41 smithi088 bash[24895]: audit 2024-09-13T00:21:40.987169+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:42.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:41 smithi077 bash[22596]: audit 2024-09-13T00:21:40.546290+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:42.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:41 smithi077 bash[22596]: audit 2024-09-13T00:21:40.550980+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:42.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:41 smithi077 bash[22596]: audit 2024-09-13T00:21:40.552587+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:42.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:41 smithi077 bash[22596]: audit 2024-09-13T00:21:40.979834+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:42.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:41 smithi077 bash[22596]: audit 2024-09-13T00:21:40.981021+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:42.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:41 smithi077 bash[22596]: audit 2024-09-13T00:21:40.987169+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:42.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:41 smithi077 bash[32040]: audit 2024-09-13T00:21:40.546290+0000 mon.a (mon.0) 237 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:42.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:41 smithi077 bash[32040]: audit 2024-09-13T00:21:40.550980+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:42.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:41 smithi077 bash[32040]: audit 2024-09-13T00:21:40.552587+0000 mon.a (mon.0) 239 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:21:42.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:41 smithi077 bash[32040]: audit 2024-09-13T00:21:40.979834+0000 mon.a (mon.0) 240 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:42.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:41 smithi077 bash[32040]: audit 2024-09-13T00:21:40.981021+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:21:42.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:41 smithi077 bash[32040]: audit 2024-09-13T00:21:40.987169+0000 mon.a (mon.0) 242 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:21:42.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:42 smithi088 bash[24895]: cluster 2024-09-13T00:21:41.175854+0000 mgr.y (mgr.14150) 84 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:43.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:42 smithi077 bash[22596]: cluster 2024-09-13T00:21:41.175854+0000 mgr.y (mgr.14150) 84 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:43.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:42 smithi077 bash[32040]: cluster 2024-09-13T00:21:41.175854+0000 mgr.y (mgr.14150) 84 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:44.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:44 smithi088 bash[24895]: cluster 2024-09-13T00:21:43.176229+0000 mgr.y (mgr.14150) 85 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:45.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:44 smithi077 bash[22596]: cluster 2024-09-13T00:21:43.176229+0000 mgr.y (mgr.14150) 85 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:45.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:44 smithi077 bash[32040]: cluster 2024-09-13T00:21:43.176229+0000 mgr.y (mgr.14150) 85 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:45.668 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:21:46.825 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:46 smithi077 bash[22596]: cluster 2024-09-13T00:21:45.176630+0000 mgr.y (mgr.14150) 86 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:46.826 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:46 smithi077 bash[32040]: cluster 2024-09-13T00:21:45.176630+0000 mgr.y (mgr.14150) 86 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:46.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:46 smithi088 bash[24895]: cluster 2024-09-13T00:21:45.176630+0000 mgr.y (mgr.14150) 86 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:48.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:48 smithi077 bash[32040]: cluster 2024-09-13T00:21:47.177036+0000 mgr.y (mgr.14150) 87 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:48.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:48 smithi077 bash[22596]: cluster 2024-09-13T00:21:47.177036+0000 mgr.y (mgr.14150) 87 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:48.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:48 smithi088 bash[24895]: cluster 2024-09-13T00:21:47.177036+0000 mgr.y (mgr.14150) 87 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:49.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:49 smithi088 bash[24895]: audit 2024-09-13T00:21:48.606461+0000 mgr.y (mgr.14150) 88 : audit [DBG] from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi077:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:21:49.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:49 smithi088 bash[24895]: audit 2024-09-13T00:21:48.612217+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:21:49.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:49 smithi088 bash[24895]: audit 2024-09-13T00:21:48.617346+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:21:49.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:49 smithi088 bash[24895]: audit 2024-09-13T00:21:48.618436+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:50.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:49 smithi077 bash[32040]: audit 2024-09-13T00:21:48.606461+0000 mgr.y (mgr.14150) 88 : audit [DBG] from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi077:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:21:50.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:49 smithi077 bash[32040]: audit 2024-09-13T00:21:48.612217+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:21:50.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:49 smithi077 bash[32040]: audit 2024-09-13T00:21:48.617346+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:21:50.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:49 smithi077 bash[32040]: audit 2024-09-13T00:21:48.618436+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:50.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:49 smithi077 bash[22596]: audit 2024-09-13T00:21:48.606461+0000 mgr.y (mgr.14150) 88 : audit [DBG] from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi077:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:21:50.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:49 smithi077 bash[22596]: audit 2024-09-13T00:21:48.612217+0000 mon.a (mon.0) 243 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:21:50.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:49 smithi077 bash[22596]: audit 2024-09-13T00:21:48.617346+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:21:50.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:49 smithi077 bash[22596]: audit 2024-09-13T00:21:48.618436+0000 mon.a (mon.0) 245 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:21:50.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:50 smithi088 bash[24895]: cluster 2024-09-13T00:21:49.177522+0000 mgr.y (mgr.14150) 89 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:51.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:50 smithi077 bash[32040]: cluster 2024-09-13T00:21:49.177522+0000 mgr.y (mgr.14150) 89 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:51.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:50 smithi077 bash[22596]: cluster 2024-09-13T00:21:49.177522+0000 mgr.y (mgr.14150) 89 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:52.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:52 smithi088 bash[24895]: cluster 2024-09-13T00:21:51.177973+0000 mgr.y (mgr.14150) 90 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:53.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:52 smithi077 bash[32040]: cluster 2024-09-13T00:21:51.177973+0000 mgr.y (mgr.14150) 90 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:53.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:52 smithi077 bash[22596]: cluster 2024-09-13T00:21:51.177973+0000 mgr.y (mgr.14150) 90 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:54.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:54 smithi088 bash[24895]: cluster 2024-09-13T00:21:53.178394+0000 mgr.y (mgr.14150) 91 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:55.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:54 smithi077 bash[22596]: cluster 2024-09-13T00:21:53.178394+0000 mgr.y (mgr.14150) 91 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:55.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:54 smithi077 bash[32040]: cluster 2024-09-13T00:21:53.178394+0000 mgr.y (mgr.14150) 91 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:56.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:56 smithi088 bash[24895]: cluster 2024-09-13T00:21:55.178787+0000 mgr.y (mgr.14150) 92 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:57.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:56 smithi077 bash[32040]: cluster 2024-09-13T00:21:55.178787+0000 mgr.y (mgr.14150) 92 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:57.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:56 smithi077 bash[22596]: cluster 2024-09-13T00:21:55.178787+0000 mgr.y (mgr.14150) 92 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:57.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:57 smithi088 bash[24895]: audit 2024-09-13T00:21:56.747040+0000 mon.a (mon.0) 246 : audit [INF] from='client.? 172.21.15.77:0/473714111' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e0860feb-72c9-4c49-933d-0a49b68caeee"}]: dispatch 2024-09-13T00:21:57.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:57 smithi088 bash[24895]: audit 2024-09-13T00:21:56.754256+0000 mon.a (mon.0) 247 : audit [INF] from='client.? 172.21.15.77:0/473714111' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e0860feb-72c9-4c49-933d-0a49b68caeee"}]': finished 2024-09-13T00:21:57.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:57 smithi088 bash[24895]: cluster 2024-09-13T00:21:56.754393+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-09-13T00:21:57.855 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:57 smithi088 bash[24895]: audit 2024-09-13T00:21:56.754564+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:21:57.855 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:57 smithi088 bash[24895]: audit 2024-09-13T00:21:57.373719+0000 mon.a (mon.0) 250 : audit [DBG] from='client.? 172.21.15.77:0/4247555544' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:21:58.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:57 smithi077 bash[32040]: audit 2024-09-13T00:21:56.747040+0000 mon.a (mon.0) 246 : audit [INF] from='client.? 172.21.15.77:0/473714111' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e0860feb-72c9-4c49-933d-0a49b68caeee"}]: dispatch 2024-09-13T00:21:58.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:57 smithi077 bash[32040]: audit 2024-09-13T00:21:56.754256+0000 mon.a (mon.0) 247 : audit [INF] from='client.? 172.21.15.77:0/473714111' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e0860feb-72c9-4c49-933d-0a49b68caeee"}]': finished 2024-09-13T00:21:58.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:57 smithi077 bash[32040]: cluster 2024-09-13T00:21:56.754393+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-09-13T00:21:58.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:57 smithi077 bash[32040]: audit 2024-09-13T00:21:56.754564+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:21:58.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:57 smithi077 bash[32040]: audit 2024-09-13T00:21:57.373719+0000 mon.a (mon.0) 250 : audit [DBG] from='client.? 172.21.15.77:0/4247555544' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:21:58.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:57 smithi077 bash[22596]: audit 2024-09-13T00:21:56.747040+0000 mon.a (mon.0) 246 : audit [INF] from='client.? 172.21.15.77:0/473714111' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e0860feb-72c9-4c49-933d-0a49b68caeee"}]: dispatch 2024-09-13T00:21:58.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:57 smithi077 bash[22596]: audit 2024-09-13T00:21:56.754256+0000 mon.a (mon.0) 247 : audit [INF] from='client.? 172.21.15.77:0/473714111' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e0860feb-72c9-4c49-933d-0a49b68caeee"}]': finished 2024-09-13T00:21:58.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:57 smithi077 bash[22596]: cluster 2024-09-13T00:21:56.754393+0000 mon.a (mon.0) 248 : cluster [DBG] osdmap e5: 1 total, 0 up, 1 in 2024-09-13T00:21:58.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:57 smithi077 bash[22596]: audit 2024-09-13T00:21:56.754564+0000 mon.a (mon.0) 249 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:21:58.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:57 smithi077 bash[22596]: audit 2024-09-13T00:21:57.373719+0000 mon.a (mon.0) 250 : audit [DBG] from='client.? 172.21.15.77:0/4247555544' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:21:59.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:21:58 smithi077 bash[22596]: cluster 2024-09-13T00:21:57.179037+0000 mgr.y (mgr.14150) 93 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:59.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:21:58 smithi077 bash[32040]: cluster 2024-09-13T00:21:57.179037+0000 mgr.y (mgr.14150) 93 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:21:59.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:21:58 smithi088 bash[24895]: cluster 2024-09-13T00:21:57.179037+0000 mgr.y (mgr.14150) 93 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:01.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:00 smithi077 bash[22596]: cluster 2024-09-13T00:21:59.179373+0000 mgr.y (mgr.14150) 94 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:01.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:00 smithi077 bash[32040]: cluster 2024-09-13T00:21:59.179373+0000 mgr.y (mgr.14150) 94 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:01.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:00 smithi088 bash[24895]: cluster 2024-09-13T00:21:59.179373+0000 mgr.y (mgr.14150) 94 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:03.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:02 smithi077 bash[22596]: cluster 2024-09-13T00:22:01.179813+0000 mgr.y (mgr.14150) 95 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:03.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:02 smithi077 bash[32040]: cluster 2024-09-13T00:22:01.179813+0000 mgr.y (mgr.14150) 95 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:03.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:02 smithi088 bash[24895]: cluster 2024-09-13T00:22:01.179813+0000 mgr.y (mgr.14150) 95 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:05.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:04 smithi077 bash[22596]: cluster 2024-09-13T00:22:03.180169+0000 mgr.y (mgr.14150) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:05.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:04 smithi077 bash[32040]: cluster 2024-09-13T00:22:03.180169+0000 mgr.y (mgr.14150) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:05.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:04 smithi088 bash[24895]: cluster 2024-09-13T00:22:03.180169+0000 mgr.y (mgr.14150) 96 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:07.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:06 smithi077 bash[22596]: cluster 2024-09-13T00:22:05.180562+0000 mgr.y (mgr.14150) 97 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:07.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:06 smithi077 bash[32040]: cluster 2024-09-13T00:22:05.180562+0000 mgr.y (mgr.14150) 97 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:07.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:06 smithi088 bash[24895]: cluster 2024-09-13T00:22:05.180562+0000 mgr.y (mgr.14150) 97 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:09.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:08 smithi077 bash[22596]: cluster 2024-09-13T00:22:07.180990+0000 mgr.y (mgr.14150) 98 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:09.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:08 smithi077 bash[32040]: cluster 2024-09-13T00:22:07.180990+0000 mgr.y (mgr.14150) 98 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:09.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:08 smithi088 bash[24895]: cluster 2024-09-13T00:22:07.180990+0000 mgr.y (mgr.14150) 98 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:10.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:09 smithi088 bash[24895]: audit 2024-09-13T00:22:09.247851+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T00:22:10.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:09 smithi088 bash[24895]: audit 2024-09-13T00:22:09.249533+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:22:10.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:09 smithi077 bash[22596]: audit 2024-09-13T00:22:09.247851+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T00:22:10.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:09 smithi077 bash[22596]: audit 2024-09-13T00:22:09.249533+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:22:10.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:09 smithi077 bash[32040]: audit 2024-09-13T00:22:09.247851+0000 mon.a (mon.0) 251 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T00:22:10.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:09 smithi077 bash[32040]: audit 2024-09-13T00:22:09.249533+0000 mon.a (mon.0) 252 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:22:10.923 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:10 smithi077 bash[22596]: cluster 2024-09-13T00:22:09.181480+0000 mgr.y (mgr.14150) 99 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:10.923 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:10 smithi077 bash[22596]: cephadm 2024-09-13T00:22:09.250846+0000 mgr.y (mgr.14150) 100 : cephadm [INF] Deploying daemon osd.0 on smithi077 2024-09-13T00:22:10.924 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:10 smithi077 bash[32040]: cluster 2024-09-13T00:22:09.181480+0000 mgr.y (mgr.14150) 99 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:10.924 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:10 smithi077 bash[32040]: cephadm 2024-09-13T00:22:09.250846+0000 mgr.y (mgr.14150) 100 : cephadm [INF] Deploying daemon osd.0 on smithi077 2024-09-13T00:22:11.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:10 smithi088 bash[24895]: cluster 2024-09-13T00:22:09.181480+0000 mgr.y (mgr.14150) 99 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:11.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:10 smithi088 bash[24895]: cephadm 2024-09-13T00:22:09.250846+0000 mgr.y (mgr.14150) 100 : cephadm [INF] Deploying daemon osd.0 on smithi077 2024-09-13T00:22:13.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:12 smithi077 bash[22596]: cluster 2024-09-13T00:22:11.181887+0000 mgr.y (mgr.14150) 101 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:13.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:12 smithi077 bash[32040]: cluster 2024-09-13T00:22:11.181887+0000 mgr.y (mgr.14150) 101 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:13.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:12 smithi088 bash[24895]: cluster 2024-09-13T00:22:11.181887+0000 mgr.y (mgr.14150) 101 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:13.719 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:13 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:22:13.720 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:22:13 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:22:13.720 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:13 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:22:14.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:13 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:22:14.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:13 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:22:14.031 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:22:13 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:22:15.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:14 smithi077 bash[22596]: cluster 2024-09-13T00:22:13.182314+0000 mgr.y (mgr.14150) 102 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:15.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:14 smithi077 bash[22596]: audit 2024-09-13T00:22:13.853032+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:22:15.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:14 smithi077 bash[22596]: audit 2024-09-13T00:22:13.857827+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:15.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:14 smithi077 bash[22596]: audit 2024-09-13T00:22:13.862183+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:15.044 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:14 smithi077 bash[32040]: cluster 2024-09-13T00:22:13.182314+0000 mgr.y (mgr.14150) 102 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:15.044 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:14 smithi077 bash[32040]: audit 2024-09-13T00:22:13.853032+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:22:15.044 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:14 smithi077 bash[32040]: audit 2024-09-13T00:22:13.857827+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:15.044 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:14 smithi077 bash[32040]: audit 2024-09-13T00:22:13.862183+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:15.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:14 smithi088 bash[24895]: cluster 2024-09-13T00:22:13.182314+0000 mgr.y (mgr.14150) 102 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:15.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:14 smithi088 bash[24895]: audit 2024-09-13T00:22:13.853032+0000 mon.a (mon.0) 253 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:22:15.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:14 smithi088 bash[24895]: audit 2024-09-13T00:22:13.857827+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:15.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:14 smithi088 bash[24895]: audit 2024-09-13T00:22:13.862183+0000 mon.a (mon.0) 255 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:17.076 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:16 smithi077 bash[22596]: cluster 2024-09-13T00:22:15.182621+0000 mgr.y (mgr.14150) 103 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:17.076 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:16 smithi077 bash[32040]: cluster 2024-09-13T00:22:15.182621+0000 mgr.y (mgr.14150) 103 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:17.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:16 smithi088 bash[24895]: cluster 2024-09-13T00:22:15.182621+0000 mgr.y (mgr.14150) 103 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:19.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:18 smithi088 bash[24895]: cluster 2024-09-13T00:22:17.182867+0000 mgr.y (mgr.14150) 104 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:19.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:18 smithi077 bash[32040]: cluster 2024-09-13T00:22:17.182867+0000 mgr.y (mgr.14150) 104 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:19.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:18 smithi077 bash[22596]: cluster 2024-09-13T00:22:17.182867+0000 mgr.y (mgr.14150) 104 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:20.972 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:20 smithi077 bash[32040]: cluster 2024-09-13T00:22:19.183263+0000 mgr.y (mgr.14150) 105 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:20.973 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:20 smithi077 bash[32040]: audit 2024-09-13T00:22:19.858764+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:20.973 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:20 smithi077 bash[32040]: audit 2024-09-13T00:22:19.864710+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:20.973 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:20 smithi077 bash[32040]: audit 2024-09-13T00:22:20.260989+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:22:20.973 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:20 smithi077 bash[32040]: audit 2024-09-13T00:22:20.262394+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:22:20.973 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:20 smithi077 bash[32040]: audit 2024-09-13T00:22:20.271384+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:20.973 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:20 smithi077 bash[32040]: audit 2024-09-13T00:22:20.681285+0000 mon.a (mon.0) 261 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/1623816886,v1:172.21.15.77:6803/1623816886]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T00:22:20.973 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:20 smithi077 bash[22596]: cluster 2024-09-13T00:22:19.183263+0000 mgr.y (mgr.14150) 105 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:20.974 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:20 smithi077 bash[22596]: audit 2024-09-13T00:22:19.858764+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:20.974 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:20 smithi077 bash[22596]: audit 2024-09-13T00:22:19.864710+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:20.974 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:20 smithi077 bash[22596]: audit 2024-09-13T00:22:20.260989+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:22:20.974 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:20 smithi077 bash[22596]: audit 2024-09-13T00:22:20.262394+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:22:20.974 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:20 smithi077 bash[22596]: audit 2024-09-13T00:22:20.271384+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:20.974 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:20 smithi077 bash[22596]: audit 2024-09-13T00:22:20.681285+0000 mon.a (mon.0) 261 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/1623816886,v1:172.21.15.77:6803/1623816886]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T00:22:21.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:20 smithi088 bash[24895]: cluster 2024-09-13T00:22:19.183263+0000 mgr.y (mgr.14150) 105 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:21.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:20 smithi088 bash[24895]: audit 2024-09-13T00:22:19.858764+0000 mon.a (mon.0) 256 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:21.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:20 smithi088 bash[24895]: audit 2024-09-13T00:22:19.864710+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:21.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:20 smithi088 bash[24895]: audit 2024-09-13T00:22:20.260989+0000 mon.a (mon.0) 258 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:22:21.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:20 smithi088 bash[24895]: audit 2024-09-13T00:22:20.262394+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:22:21.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:20 smithi088 bash[24895]: audit 2024-09-13T00:22:20.271384+0000 mon.a (mon.0) 260 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:21.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:20 smithi088 bash[24895]: audit 2024-09-13T00:22:20.681285+0000 mon.a (mon.0) 261 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/1623816886,v1:172.21.15.77:6803/1623816886]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T00:22:22.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:22 smithi077 bash[22596]: cluster 2024-09-13T00:22:21.183695+0000 mgr.y (mgr.14150) 106 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:22.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:22 smithi077 bash[22596]: audit 2024-09-13T00:22:21.273977+0000 mon.a (mon.0) 262 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/1623816886,v1:172.21.15.77:6803/1623816886]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T00:22:22.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:22 smithi077 bash[22596]: cluster 2024-09-13T00:22:21.274081+0000 mon.a (mon.0) 263 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-09-13T00:22:22.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:22 smithi077 bash[22596]: audit 2024-09-13T00:22:21.274304+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:22:22.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:22 smithi077 bash[22596]: audit 2024-09-13T00:22:21.275051+0000 mon.a (mon.0) 265 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/1623816886,v1:172.21.15.77:6803/1623816886]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:22:22.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:22 smithi077 bash[32040]: cluster 2024-09-13T00:22:21.183695+0000 mgr.y (mgr.14150) 106 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:22.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:22 smithi077 bash[32040]: audit 2024-09-13T00:22:21.273977+0000 mon.a (mon.0) 262 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/1623816886,v1:172.21.15.77:6803/1623816886]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T00:22:22.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:22 smithi077 bash[32040]: cluster 2024-09-13T00:22:21.274081+0000 mon.a (mon.0) 263 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-09-13T00:22:22.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:22 smithi077 bash[32040]: audit 2024-09-13T00:22:21.274304+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:22:22.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:22 smithi077 bash[32040]: audit 2024-09-13T00:22:21.275051+0000 mon.a (mon.0) 265 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/1623816886,v1:172.21.15.77:6803/1623816886]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:22:22.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:22 smithi088 bash[24895]: cluster 2024-09-13T00:22:21.183695+0000 mgr.y (mgr.14150) 106 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:22 smithi088 bash[24895]: audit 2024-09-13T00:22:21.273977+0000 mon.a (mon.0) 262 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/1623816886,v1:172.21.15.77:6803/1623816886]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T00:22:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:22 smithi088 bash[24895]: cluster 2024-09-13T00:22:21.274081+0000 mon.a (mon.0) 263 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2024-09-13T00:22:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:22 smithi088 bash[24895]: audit 2024-09-13T00:22:21.274304+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:22:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:22 smithi088 bash[24895]: audit 2024-09-13T00:22:21.275051+0000 mon.a (mon.0) 265 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/1623816886,v1:172.21.15.77:6803/1623816886]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:22:23.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:23 smithi088 bash[24895]: audit 2024-09-13T00:22:22.276195+0000 mon.a (mon.0) 266 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/1623816886,v1:172.21.15.77:6803/1623816886]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]': finished 2024-09-13T00:22:23.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:23 smithi088 bash[24895]: cluster 2024-09-13T00:22:22.276332+0000 mon.a (mon.0) 267 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-09-13T00:22:23.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:23 smithi088 bash[24895]: audit 2024-09-13T00:22:22.277049+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:22:23.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:23 smithi088 bash[24895]: audit 2024-09-13T00:22:22.287357+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:22:23.768 INFO:teuthology.orchestra.run.smithi077.stdout:Created osd(s) 0 on host 'smithi077' 2024-09-13T00:22:23.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:23 smithi077 bash[22596]: audit 2024-09-13T00:22:22.276195+0000 mon.a (mon.0) 266 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/1623816886,v1:172.21.15.77:6803/1623816886]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]': finished 2024-09-13T00:22:23.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:23 smithi077 bash[22596]: cluster 2024-09-13T00:22:22.276332+0000 mon.a (mon.0) 267 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-09-13T00:22:23.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:23 smithi077 bash[22596]: audit 2024-09-13T00:22:22.277049+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:22:23.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:23 smithi077 bash[22596]: audit 2024-09-13T00:22:22.287357+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:22:23.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:23 smithi077 bash[32040]: audit 2024-09-13T00:22:22.276195+0000 mon.a (mon.0) 266 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/1623816886,v1:172.21.15.77:6803/1623816886]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]': finished 2024-09-13T00:22:23.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:23 smithi077 bash[32040]: cluster 2024-09-13T00:22:22.276332+0000 mon.a (mon.0) 267 : cluster [DBG] osdmap e7: 1 total, 0 up, 1 in 2024-09-13T00:22:23.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:23 smithi077 bash[32040]: audit 2024-09-13T00:22:22.277049+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:22:23.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:23 smithi077 bash[32040]: audit 2024-09-13T00:22:22.287357+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:22:24.511 DEBUG:teuthology.orchestra.run.smithi077:osd.0> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.0.service 2024-09-13T00:22:24.515 INFO:tasks.cephadm:Deploying osd.1 on smithi077 with /dev/vg_nvme/lv_3... 2024-09-13T00:22:24.515 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-09-13T00:22:24.566 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[22596]: cluster 2024-09-13T00:22:21.643816+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:22:24.566 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[22596]: cluster 2024-09-13T00:22:21.643904+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:22:24.567 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[22596]: cluster 2024-09-13T00:22:23.184115+0000 mgr.y (mgr.14150) 107 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:24.567 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[22596]: cluster 2024-09-13T00:22:23.289202+0000 mon.a (mon.0) 270 : cluster [INF] osd.0 [v2:172.21.15.77:6802/1623816886,v1:172.21.15.77:6803/1623816886] boot 2024-09-13T00:22:24.567 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[22596]: cluster 2024-09-13T00:22:23.289279+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-09-13T00:22:24.567 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[22596]: audit 2024-09-13T00:22:23.290109+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:22:24.567 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[22596]: audit 2024-09-13T00:22:23.742188+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:22:24.567 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[22596]: audit 2024-09-13T00:22:23.751120+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:24.567 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[22596]: audit 2024-09-13T00:22:23.761399+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:24.567 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[32040]: cluster 2024-09-13T00:22:21.643816+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:22:24.567 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[32040]: cluster 2024-09-13T00:22:21.643904+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:22:24.567 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[32040]: cluster 2024-09-13T00:22:23.184115+0000 mgr.y (mgr.14150) 107 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:24.568 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[32040]: cluster 2024-09-13T00:22:23.289202+0000 mon.a (mon.0) 270 : cluster [INF] osd.0 [v2:172.21.15.77:6802/1623816886,v1:172.21.15.77:6803/1623816886] boot 2024-09-13T00:22:24.568 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[32040]: cluster 2024-09-13T00:22:23.289279+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-09-13T00:22:24.568 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[32040]: audit 2024-09-13T00:22:23.290109+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:22:24.568 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[32040]: audit 2024-09-13T00:22:23.742188+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:22:24.568 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[32040]: audit 2024-09-13T00:22:23.751120+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:24.568 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:24 smithi077 bash[32040]: audit 2024-09-13T00:22:23.761399+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:24.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:24 smithi088 bash[24895]: cluster 2024-09-13T00:22:21.643816+0000 osd.0 (osd.0) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:22:24.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:24 smithi088 bash[24895]: cluster 2024-09-13T00:22:21.643904+0000 osd.0 (osd.0) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:22:24.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:24 smithi088 bash[24895]: cluster 2024-09-13T00:22:23.184115+0000 mgr.y (mgr.14150) 107 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-13T00:22:24.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:24 smithi088 bash[24895]: cluster 2024-09-13T00:22:23.289202+0000 mon.a (mon.0) 270 : cluster [INF] osd.0 [v2:172.21.15.77:6802/1623816886,v1:172.21.15.77:6803/1623816886] boot 2024-09-13T00:22:24.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:24 smithi088 bash[24895]: cluster 2024-09-13T00:22:23.289279+0000 mon.a (mon.0) 271 : cluster [DBG] osdmap e8: 1 total, 1 up, 1 in 2024-09-13T00:22:24.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:24 smithi088 bash[24895]: audit 2024-09-13T00:22:23.290109+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:22:24.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:24 smithi088 bash[24895]: audit 2024-09-13T00:22:23.742188+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:22:24.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:24 smithi088 bash[24895]: audit 2024-09-13T00:22:23.751120+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:24.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:24 smithi088 bash[24895]: audit 2024-09-13T00:22:23.761399+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:25.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:25 smithi088 bash[24895]: cluster 2024-09-13T00:22:24.304210+0000 mon.a (mon.0) 276 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-09-13T00:22:25.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:25 smithi077 bash[32040]: cluster 2024-09-13T00:22:24.304210+0000 mon.a (mon.0) 276 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-09-13T00:22:25.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:25 smithi077 bash[22596]: cluster 2024-09-13T00:22:24.304210+0000 mon.a (mon.0) 276 : cluster [DBG] osdmap e9: 1 total, 1 up, 1 in 2024-09-13T00:22:26.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:26 smithi088 bash[24895]: cluster 2024-09-13T00:22:25.184690+0000 mgr.y (mgr.14150) 108 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:26.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:26 smithi077 bash[22596]: cluster 2024-09-13T00:22:25.184690+0000 mgr.y (mgr.14150) 108 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:26.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:26 smithi077 bash[32040]: cluster 2024-09-13T00:22:25.184690+0000 mgr.y (mgr.14150) 108 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:28.518 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:22:28.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:28 smithi088 bash[24895]: cluster 2024-09-13T00:22:27.185284+0000 mgr.y (mgr.14150) 109 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:28.657 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:28 smithi077 bash[22596]: cluster 2024-09-13T00:22:27.185284+0000 mgr.y (mgr.14150) 109 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:28.657 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:28 smithi077 bash[32040]: cluster 2024-09-13T00:22:27.185284+0000 mgr.y (mgr.14150) 109 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:30.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:30 smithi077 bash[32040]: cluster 2024-09-13T00:22:29.185805+0000 mgr.y (mgr.14150) 110 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:30.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:30 smithi077 bash[22596]: cluster 2024-09-13T00:22:29.185805+0000 mgr.y (mgr.14150) 110 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:30.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:30 smithi088 bash[24895]: cluster 2024-09-13T00:22:29.185805+0000 mgr.y (mgr.14150) 110 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:32.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:32 smithi088 bash[24895]: cluster 2024-09-13T00:22:31.186369+0000 mgr.y (mgr.14150) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:32.786 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:32 smithi077 bash[22596]: cluster 2024-09-13T00:22:31.186369+0000 mgr.y (mgr.14150) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:32.787 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:32 smithi077 bash[32040]: cluster 2024-09-13T00:22:31.186369+0000 mgr.y (mgr.14150) 111 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:33.469 INFO:teuthology.orchestra.run.smithi077.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-09-13T00:22:33.650 INFO:teuthology.orchestra.run.smithi077.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-09-13T00:22:33.666 INFO:teuthology.orchestra.run.smithi077.stderr: stderr: 10+0 records in 2024-09-13T00:22:33.666 INFO:teuthology.orchestra.run.smithi077.stderr:10+0 records out 2024-09-13T00:22:33.666 INFO:teuthology.orchestra.run.smithi077.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0139315 s, 753 MB/s 2024-09-13T00:22:33.666 INFO:teuthology.orchestra.run.smithi077.stderr:--> Zapping successful for: 2024-09-13T00:22:34.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:34 smithi088 bash[24895]: cluster 2024-09-13T00:22:33.186705+0000 mgr.y (mgr.14150) 112 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:34.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:34 smithi077 bash[32040]: cluster 2024-09-13T00:22:33.186705+0000 mgr.y (mgr.14150) 112 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:34.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:34 smithi077 bash[22596]: cluster 2024-09-13T00:22:33.186705+0000 mgr.y (mgr.14150) 112 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:34.855 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi077:vg_nvme/lv_3 2024-09-13T00:22:36.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[22596]: cephadm 2024-09-13T00:22:34.938801+0000 mgr.y (mgr.14150) 113 : cephadm [INF] Detected new or changed devices on smithi077 2024-09-13T00:22:36.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[22596]: audit 2024-09-13T00:22:34.944810+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:36.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[22596]: audit 2024-09-13T00:22:34.952518+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:36.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[22596]: audit 2024-09-13T00:22:34.954152+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:22:36.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[22596]: cephadm 2024-09-13T00:22:34.955188+0000 mgr.y (mgr.14150) 114 : cephadm [INF] Adjusting osd_memory_target on smithi077 to 16241M 2024-09-13T00:22:36.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[22596]: audit 2024-09-13T00:22:34.960361+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:36.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[22596]: audit 2024-09-13T00:22:34.962944+0000 mon.a (mon.0) 281 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:22:36.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[22596]: audit 2024-09-13T00:22:34.963988+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:22:36.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[22596]: audit 2024-09-13T00:22:34.970621+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:36.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[22596]: cluster 2024-09-13T00:22:35.187159+0000 mgr.y (mgr.14150) 115 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:36.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[32040]: cephadm 2024-09-13T00:22:34.938801+0000 mgr.y (mgr.14150) 113 : cephadm [INF] Detected new or changed devices on smithi077 2024-09-13T00:22:36.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[32040]: audit 2024-09-13T00:22:34.944810+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:36.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[32040]: audit 2024-09-13T00:22:34.952518+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:36.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[32040]: audit 2024-09-13T00:22:34.954152+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:22:36.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[32040]: cephadm 2024-09-13T00:22:34.955188+0000 mgr.y (mgr.14150) 114 : cephadm [INF] Adjusting osd_memory_target on smithi077 to 16241M 2024-09-13T00:22:36.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[32040]: audit 2024-09-13T00:22:34.960361+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:36.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[32040]: audit 2024-09-13T00:22:34.962944+0000 mon.a (mon.0) 281 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:22:36.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[32040]: audit 2024-09-13T00:22:34.963988+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:22:36.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[32040]: audit 2024-09-13T00:22:34.970621+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:36.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:35 smithi077 bash[32040]: cluster 2024-09-13T00:22:35.187159+0000 mgr.y (mgr.14150) 115 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:36.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:35 smithi088 bash[24895]: cephadm 2024-09-13T00:22:34.938801+0000 mgr.y (mgr.14150) 113 : cephadm [INF] Detected new or changed devices on smithi077 2024-09-13T00:22:36.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:35 smithi088 bash[24895]: audit 2024-09-13T00:22:34.944810+0000 mon.a (mon.0) 277 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:36.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:35 smithi088 bash[24895]: audit 2024-09-13T00:22:34.952518+0000 mon.a (mon.0) 278 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:36.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:35 smithi088 bash[24895]: audit 2024-09-13T00:22:34.954152+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:22:36.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:35 smithi088 bash[24895]: cephadm 2024-09-13T00:22:34.955188+0000 mgr.y (mgr.14150) 114 : cephadm [INF] Adjusting osd_memory_target on smithi077 to 16241M 2024-09-13T00:22:36.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:35 smithi088 bash[24895]: audit 2024-09-13T00:22:34.960361+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:36.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:35 smithi088 bash[24895]: audit 2024-09-13T00:22:34.962944+0000 mon.a (mon.0) 281 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:22:36.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:35 smithi088 bash[24895]: audit 2024-09-13T00:22:34.963988+0000 mon.a (mon.0) 282 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:22:36.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:35 smithi088 bash[24895]: audit 2024-09-13T00:22:34.970621+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:22:36.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:35 smithi088 bash[24895]: cluster 2024-09-13T00:22:35.187159+0000 mgr.y (mgr.14150) 115 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:38.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:38 smithi077 bash[22596]: cluster 2024-09-13T00:22:37.187616+0000 mgr.y (mgr.14150) 116 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:38.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:38 smithi077 bash[32040]: cluster 2024-09-13T00:22:37.187616+0000 mgr.y (mgr.14150) 116 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:38.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:38 smithi088 bash[24895]: cluster 2024-09-13T00:22:37.187616+0000 mgr.y (mgr.14150) 116 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:39.638 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:22:40.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:40 smithi077 bash[22596]: cluster 2024-09-13T00:22:39.188063+0000 mgr.y (mgr.14150) 117 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:40.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:40 smithi077 bash[32040]: cluster 2024-09-13T00:22:39.188063+0000 mgr.y (mgr.14150) 117 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:40.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:40 smithi088 bash[24895]: cluster 2024-09-13T00:22:39.188063+0000 mgr.y (mgr.14150) 117 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:42.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:42 smithi077 bash[32040]: cluster 2024-09-13T00:22:41.188551+0000 mgr.y (mgr.14150) 118 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:42.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:42 smithi077 bash[22596]: cluster 2024-09-13T00:22:41.188551+0000 mgr.y (mgr.14150) 118 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:42.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:42 smithi088 bash[24895]: cluster 2024-09-13T00:22:41.188551+0000 mgr.y (mgr.14150) 118 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:43.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:43 smithi077 bash[22596]: audit 2024-09-13T00:22:43.090619+0000 mon.a (mon.0) 284 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:22:43.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:43 smithi077 bash[22596]: audit 2024-09-13T00:22:43.095754+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:22:43.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:43 smithi077 bash[22596]: audit 2024-09-13T00:22:43.097027+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:22:43.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:43 smithi077 bash[32040]: audit 2024-09-13T00:22:43.090619+0000 mon.a (mon.0) 284 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:22:43.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:43 smithi077 bash[32040]: audit 2024-09-13T00:22:43.095754+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:22:43.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:43 smithi077 bash[32040]: audit 2024-09-13T00:22:43.097027+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:22:43.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:43 smithi088 bash[24895]: audit 2024-09-13T00:22:43.090619+0000 mon.a (mon.0) 284 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:22:43.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:43 smithi088 bash[24895]: audit 2024-09-13T00:22:43.095754+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:22:43.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:43 smithi088 bash[24895]: audit 2024-09-13T00:22:43.097027+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:22:44.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:44 smithi077 bash[22596]: audit 2024-09-13T00:22:43.087891+0000 mgr.y (mgr.14150) 119 : audit [DBG] from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi077:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:22:44.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:44 smithi077 bash[22596]: cluster 2024-09-13T00:22:43.188905+0000 mgr.y (mgr.14150) 120 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:44.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:44 smithi077 bash[32040]: audit 2024-09-13T00:22:43.087891+0000 mgr.y (mgr.14150) 119 : audit [DBG] from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi077:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:22:44.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:44 smithi077 bash[32040]: cluster 2024-09-13T00:22:43.188905+0000 mgr.y (mgr.14150) 120 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:44.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:44 smithi088 bash[24895]: audit 2024-09-13T00:22:43.087891+0000 mgr.y (mgr.14150) 119 : audit [DBG] from='client.14238 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi077:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:22:44.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:44 smithi088 bash[24895]: cluster 2024-09-13T00:22:43.188905+0000 mgr.y (mgr.14150) 120 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:46.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:46 smithi077 bash[22596]: cluster 2024-09-13T00:22:45.189299+0000 mgr.y (mgr.14150) 121 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:46.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:46 smithi077 bash[32040]: cluster 2024-09-13T00:22:45.189299+0000 mgr.y (mgr.14150) 121 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:46.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:46 smithi088 bash[24895]: cluster 2024-09-13T00:22:45.189299+0000 mgr.y (mgr.14150) 121 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:48.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:48 smithi077 bash[22596]: cluster 2024-09-13T00:22:47.189745+0000 mgr.y (mgr.14150) 122 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:48.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:48 smithi077 bash[32040]: cluster 2024-09-13T00:22:47.189745+0000 mgr.y (mgr.14150) 122 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:48.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:48 smithi088 bash[24895]: cluster 2024-09-13T00:22:47.189745+0000 mgr.y (mgr.14150) 122 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:50.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:50 smithi077 bash[22596]: cluster 2024-09-13T00:22:49.190108+0000 mgr.y (mgr.14150) 123 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:50.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:50 smithi077 bash[32040]: cluster 2024-09-13T00:22:49.190108+0000 mgr.y (mgr.14150) 123 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:50.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:50 smithi088 bash[24895]: cluster 2024-09-13T00:22:49.190108+0000 mgr.y (mgr.14150) 123 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:52.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:52 smithi077 bash[32040]: cluster 2024-09-13T00:22:51.190531+0000 mgr.y (mgr.14150) 124 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:52.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:52 smithi077 bash[32040]: audit 2024-09-13T00:22:51.742870+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.77:0/1422039217' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7ee01388-61b0-4a60-81f2-08c3c072ec0e"}]: dispatch 2024-09-13T00:22:52.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:52 smithi077 bash[32040]: audit 2024-09-13T00:22:51.743467+0000 mon.a (mon.0) 287 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7ee01388-61b0-4a60-81f2-08c3c072ec0e"}]: dispatch 2024-09-13T00:22:52.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:52 smithi077 bash[32040]: audit 2024-09-13T00:22:51.748716+0000 mon.a (mon.0) 288 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7ee01388-61b0-4a60-81f2-08c3c072ec0e"}]': finished 2024-09-13T00:22:52.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:52 smithi077 bash[32040]: cluster 2024-09-13T00:22:51.748752+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-09-13T00:22:52.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:52 smithi077 bash[32040]: audit 2024-09-13T00:22:51.748916+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:22:52.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:52 smithi077 bash[22596]: cluster 2024-09-13T00:22:51.190531+0000 mgr.y (mgr.14150) 124 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:52.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:52 smithi077 bash[22596]: audit 2024-09-13T00:22:51.742870+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.77:0/1422039217' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7ee01388-61b0-4a60-81f2-08c3c072ec0e"}]: dispatch 2024-09-13T00:22:52.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:52 smithi077 bash[22596]: audit 2024-09-13T00:22:51.743467+0000 mon.a (mon.0) 287 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7ee01388-61b0-4a60-81f2-08c3c072ec0e"}]: dispatch 2024-09-13T00:22:52.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:52 smithi077 bash[22596]: audit 2024-09-13T00:22:51.748716+0000 mon.a (mon.0) 288 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7ee01388-61b0-4a60-81f2-08c3c072ec0e"}]': finished 2024-09-13T00:22:52.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:52 smithi077 bash[22596]: cluster 2024-09-13T00:22:51.748752+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-09-13T00:22:52.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:52 smithi077 bash[22596]: audit 2024-09-13T00:22:51.748916+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:22:52.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:52 smithi088 bash[24895]: cluster 2024-09-13T00:22:51.190531+0000 mgr.y (mgr.14150) 124 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:52.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:52 smithi088 bash[24895]: audit 2024-09-13T00:22:51.742870+0000 mon.c (mon.2) 2 : audit [INF] from='client.? 172.21.15.77:0/1422039217' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7ee01388-61b0-4a60-81f2-08c3c072ec0e"}]: dispatch 2024-09-13T00:22:52.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:52 smithi088 bash[24895]: audit 2024-09-13T00:22:51.743467+0000 mon.a (mon.0) 287 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "7ee01388-61b0-4a60-81f2-08c3c072ec0e"}]: dispatch 2024-09-13T00:22:52.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:52 smithi088 bash[24895]: audit 2024-09-13T00:22:51.748716+0000 mon.a (mon.0) 288 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "7ee01388-61b0-4a60-81f2-08c3c072ec0e"}]': finished 2024-09-13T00:22:52.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:52 smithi088 bash[24895]: cluster 2024-09-13T00:22:51.748752+0000 mon.a (mon.0) 289 : cluster [DBG] osdmap e10: 2 total, 1 up, 2 in 2024-09-13T00:22:52.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:52 smithi088 bash[24895]: audit 2024-09-13T00:22:51.748916+0000 mon.a (mon.0) 290 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:22:53.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:53 smithi088 bash[24895]: audit 2024-09-13T00:22:52.328926+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.77:0/1903618422' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:22:53.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:53 smithi077 bash[22596]: audit 2024-09-13T00:22:52.328926+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.77:0/1903618422' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:22:53.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:53 smithi077 bash[32040]: audit 2024-09-13T00:22:52.328926+0000 mon.c (mon.2) 3 : audit [DBG] from='client.? 172.21.15.77:0/1903618422' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:22:54.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:54 smithi088 bash[24895]: cluster 2024-09-13T00:22:53.190945+0000 mgr.y (mgr.14150) 125 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:54.734 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:54 smithi077 bash[32040]: cluster 2024-09-13T00:22:53.190945+0000 mgr.y (mgr.14150) 125 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:54.734 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:54 smithi077 bash[22596]: cluster 2024-09-13T00:22:53.190945+0000 mgr.y (mgr.14150) 125 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:56.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:56 smithi088 bash[24895]: cluster 2024-09-13T00:22:55.191337+0000 mgr.y (mgr.14150) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:56.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:56 smithi077 bash[32040]: cluster 2024-09-13T00:22:55.191337+0000 mgr.y (mgr.14150) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:56.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:56 smithi077 bash[22596]: cluster 2024-09-13T00:22:55.191337+0000 mgr.y (mgr.14150) 126 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:58.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:22:58 smithi088 bash[24895]: cluster 2024-09-13T00:22:57.191789+0000 mgr.y (mgr.14150) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:58.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:22:58 smithi077 bash[32040]: cluster 2024-09-13T00:22:57.191789+0000 mgr.y (mgr.14150) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:22:58.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:22:58 smithi077 bash[22596]: cluster 2024-09-13T00:22:57.191789+0000 mgr.y (mgr.14150) 127 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:00.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:00 smithi088 bash[24895]: cluster 2024-09-13T00:22:59.192224+0000 mgr.y (mgr.14150) 128 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:00.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:00 smithi077 bash[22596]: cluster 2024-09-13T00:22:59.192224+0000 mgr.y (mgr.14150) 128 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:00.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:00 smithi077 bash[32040]: cluster 2024-09-13T00:22:59.192224+0000 mgr.y (mgr.14150) 128 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:02.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:02 smithi088 bash[24895]: cluster 2024-09-13T00:23:01.192663+0000 mgr.y (mgr.14150) 129 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:02.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:02 smithi077 bash[22596]: cluster 2024-09-13T00:23:01.192663+0000 mgr.y (mgr.14150) 129 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:02.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:02 smithi077 bash[32040]: cluster 2024-09-13T00:23:01.192663+0000 mgr.y (mgr.14150) 129 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:04.574 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:04 smithi077 bash[22596]: cluster 2024-09-13T00:23:03.193113+0000 mgr.y (mgr.14150) 130 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:04.574 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:04 smithi077 bash[32040]: cluster 2024-09-13T00:23:03.193113+0000 mgr.y (mgr.14150) 130 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:04.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:04 smithi088 bash[24895]: cluster 2024-09-13T00:23:03.193113+0000 mgr.y (mgr.14150) 130 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:05.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:05 smithi088 bash[24895]: audit 2024-09-13T00:23:04.356799+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T00:23:05.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:05 smithi088 bash[24895]: audit 2024-09-13T00:23:04.358283+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:23:05.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:05 smithi088 bash[24895]: cephadm 2024-09-13T00:23:04.359669+0000 mgr.y (mgr.14150) 131 : cephadm [INF] Deploying daemon osd.1 on smithi077 2024-09-13T00:23:05.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:05 smithi077 bash[22596]: audit 2024-09-13T00:23:04.356799+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T00:23:05.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:05 smithi077 bash[22596]: audit 2024-09-13T00:23:04.358283+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:23:05.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:05 smithi077 bash[22596]: cephadm 2024-09-13T00:23:04.359669+0000 mgr.y (mgr.14150) 131 : cephadm [INF] Deploying daemon osd.1 on smithi077 2024-09-13T00:23:05.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:05 smithi077 bash[32040]: audit 2024-09-13T00:23:04.356799+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T00:23:05.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:05 smithi077 bash[32040]: audit 2024-09-13T00:23:04.358283+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:23:05.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:05 smithi077 bash[32040]: cephadm 2024-09-13T00:23:04.359669+0000 mgr.y (mgr.14150) 131 : cephadm [INF] Deploying daemon osd.1 on smithi077 2024-09-13T00:23:06.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:06 smithi077 bash[22596]: cluster 2024-09-13T00:23:05.193545+0000 mgr.y (mgr.14150) 132 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:06.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:06 smithi077 bash[32040]: cluster 2024-09-13T00:23:05.193545+0000 mgr.y (mgr.14150) 132 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:06.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:06 smithi088 bash[24895]: cluster 2024-09-13T00:23:05.193545+0000 mgr.y (mgr.14150) 132 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:08.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:08 smithi077 bash[22596]: cluster 2024-09-13T00:23:07.193982+0000 mgr.y (mgr.14150) 133 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:08.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:08 smithi077 bash[32040]: cluster 2024-09-13T00:23:07.193982+0000 mgr.y (mgr.14150) 133 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:08.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:08 smithi088 bash[24895]: cluster 2024-09-13T00:23:07.193982+0000 mgr.y (mgr.14150) 133 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:09.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:08 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:23:09.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:08 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:23:09.031 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:23:08 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:23:09.031 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:23:08 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:23:09.530 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:23:09 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:23:09.531 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:23:09 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:23:09.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:09 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:23:09.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:09 smithi077 bash[22596]: audit 2024-09-13T00:23:09.231495+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:23:09.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:09 smithi077 bash[22596]: audit 2024-09-13T00:23:09.236652+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:09.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:09 smithi077 bash[22596]: audit 2024-09-13T00:23:09.241219+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:09.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:09 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:23:09.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:09 smithi077 bash[32040]: audit 2024-09-13T00:23:09.231495+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:23:09.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:09 smithi077 bash[32040]: audit 2024-09-13T00:23:09.236652+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:09.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:09 smithi077 bash[32040]: audit 2024-09-13T00:23:09.241219+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:09.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:09 smithi088 bash[24895]: audit 2024-09-13T00:23:09.231495+0000 mon.a (mon.0) 293 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:23:09.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:09 smithi088 bash[24895]: audit 2024-09-13T00:23:09.236652+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:09.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:09 smithi088 bash[24895]: audit 2024-09-13T00:23:09.241219+0000 mon.a (mon.0) 295 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:10.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:10 smithi088 bash[24895]: cluster 2024-09-13T00:23:09.194415+0000 mgr.y (mgr.14150) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:10.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:10 smithi077 bash[22596]: cluster 2024-09-13T00:23:09.194415+0000 mgr.y (mgr.14150) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:10.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:10 smithi077 bash[32040]: cluster 2024-09-13T00:23:09.194415+0000 mgr.y (mgr.14150) 134 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:12.590 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:12 smithi077 bash[32040]: cluster 2024-09-13T00:23:11.194898+0000 mgr.y (mgr.14150) 135 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:12.591 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:12 smithi077 bash[22596]: cluster 2024-09-13T00:23:11.194898+0000 mgr.y (mgr.14150) 135 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:12.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:12 smithi088 bash[24895]: cluster 2024-09-13T00:23:11.194898+0000 mgr.y (mgr.14150) 135 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:14.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:14 smithi088 bash[24895]: cluster 2024-09-13T00:23:13.195316+0000 mgr.y (mgr.14150) 136 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:14.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:14 smithi077 bash[22596]: cluster 2024-09-13T00:23:13.195316+0000 mgr.y (mgr.14150) 136 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:14.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:14 smithi077 bash[32040]: cluster 2024-09-13T00:23:13.195316+0000 mgr.y (mgr.14150) 136 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:16.372 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:16 smithi077 bash[22596]: cluster 2024-09-13T00:23:15.195593+0000 mgr.y (mgr.14150) 137 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:16.372 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:16 smithi077 bash[22596]: audit 2024-09-13T00:23:15.330943+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:16.373 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:16 smithi077 bash[22596]: audit 2024-09-13T00:23:15.337111+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:16.373 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:16 smithi077 bash[22596]: audit 2024-09-13T00:23:15.337917+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:23:16.373 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:16 smithi077 bash[22596]: audit 2024-09-13T00:23:15.338669+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:23:16.373 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:16 smithi077 bash[22596]: audit 2024-09-13T00:23:15.345700+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:16.373 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:16 smithi077 bash[22596]: audit 2024-09-13T00:23:16.114659+0000 mon.a (mon.0) 301 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/1691613782,v1:172.21.15.77:6811/1691613782]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T00:23:16.373 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:16 smithi077 bash[32040]: cluster 2024-09-13T00:23:15.195593+0000 mgr.y (mgr.14150) 137 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:16.373 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:16 smithi077 bash[32040]: audit 2024-09-13T00:23:15.330943+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:16.373 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:16 smithi077 bash[32040]: audit 2024-09-13T00:23:15.337111+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:16.373 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:16 smithi077 bash[32040]: audit 2024-09-13T00:23:15.337917+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:23:16.373 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:16 smithi077 bash[32040]: audit 2024-09-13T00:23:15.338669+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:23:16.373 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:16 smithi077 bash[32040]: audit 2024-09-13T00:23:15.345700+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:16.374 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:16 smithi077 bash[32040]: audit 2024-09-13T00:23:16.114659+0000 mon.a (mon.0) 301 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/1691613782,v1:172.21.15.77:6811/1691613782]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T00:23:16.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:16 smithi088 bash[24895]: cluster 2024-09-13T00:23:15.195593+0000 mgr.y (mgr.14150) 137 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:16.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:16 smithi088 bash[24895]: audit 2024-09-13T00:23:15.330943+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:16.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:16 smithi088 bash[24895]: audit 2024-09-13T00:23:15.337111+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:16.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:16 smithi088 bash[24895]: audit 2024-09-13T00:23:15.337917+0000 mon.a (mon.0) 298 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:23:16.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:16 smithi088 bash[24895]: audit 2024-09-13T00:23:15.338669+0000 mon.a (mon.0) 299 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:23:16.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:16 smithi088 bash[24895]: audit 2024-09-13T00:23:15.345700+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:16.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:16 smithi088 bash[24895]: audit 2024-09-13T00:23:16.114659+0000 mon.a (mon.0) 301 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/1691613782,v1:172.21.15.77:6811/1691613782]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T00:23:17.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:17 smithi077 bash[32040]: audit 2024-09-13T00:23:16.348215+0000 mon.a (mon.0) 302 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/1691613782,v1:172.21.15.77:6811/1691613782]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T00:23:17.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:17 smithi077 bash[32040]: cluster 2024-09-13T00:23:16.348263+0000 mon.a (mon.0) 303 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-09-13T00:23:17.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:17 smithi077 bash[32040]: audit 2024-09-13T00:23:16.348385+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:23:17.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:17 smithi077 bash[32040]: audit 2024-09-13T00:23:16.348749+0000 mon.a (mon.0) 305 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/1691613782,v1:172.21.15.77:6811/1691613782]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:23:17.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:17 smithi077 bash[22596]: audit 2024-09-13T00:23:16.348215+0000 mon.a (mon.0) 302 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/1691613782,v1:172.21.15.77:6811/1691613782]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T00:23:17.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:17 smithi077 bash[22596]: cluster 2024-09-13T00:23:16.348263+0000 mon.a (mon.0) 303 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-09-13T00:23:17.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:17 smithi077 bash[22596]: audit 2024-09-13T00:23:16.348385+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:23:17.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:17 smithi077 bash[22596]: audit 2024-09-13T00:23:16.348749+0000 mon.a (mon.0) 305 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/1691613782,v1:172.21.15.77:6811/1691613782]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:23:17.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:17 smithi088 bash[24895]: audit 2024-09-13T00:23:16.348215+0000 mon.a (mon.0) 302 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/1691613782,v1:172.21.15.77:6811/1691613782]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T00:23:17.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:17 smithi088 bash[24895]: cluster 2024-09-13T00:23:16.348263+0000 mon.a (mon.0) 303 : cluster [DBG] osdmap e11: 2 total, 1 up, 2 in 2024-09-13T00:23:17.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:17 smithi088 bash[24895]: audit 2024-09-13T00:23:16.348385+0000 mon.a (mon.0) 304 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:23:17.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:17 smithi088 bash[24895]: audit 2024-09-13T00:23:16.348749+0000 mon.a (mon.0) 305 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/1691613782,v1:172.21.15.77:6811/1691613782]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:23:18.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:18 smithi077 bash[22596]: cluster 2024-09-13T00:23:17.196016+0000 mgr.y (mgr.14150) 138 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:18.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:18 smithi077 bash[22596]: audit 2024-09-13T00:23:17.354196+0000 mon.a (mon.0) 306 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/1691613782,v1:172.21.15.77:6811/1691613782]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]': finished 2024-09-13T00:23:18.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:18 smithi077 bash[22596]: cluster 2024-09-13T00:23:17.354354+0000 mon.a (mon.0) 307 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-09-13T00:23:18.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:18 smithi077 bash[22596]: audit 2024-09-13T00:23:17.355240+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:23:18.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:18 smithi077 bash[22596]: audit 2024-09-13T00:23:17.366431+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:23:18.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:18 smithi077 bash[32040]: cluster 2024-09-13T00:23:17.196016+0000 mgr.y (mgr.14150) 138 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:18.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:18 smithi077 bash[32040]: audit 2024-09-13T00:23:17.354196+0000 mon.a (mon.0) 306 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/1691613782,v1:172.21.15.77:6811/1691613782]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]': finished 2024-09-13T00:23:18.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:18 smithi077 bash[32040]: cluster 2024-09-13T00:23:17.354354+0000 mon.a (mon.0) 307 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-09-13T00:23:18.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:18 smithi077 bash[32040]: audit 2024-09-13T00:23:17.355240+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:23:18.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:18 smithi077 bash[32040]: audit 2024-09-13T00:23:17.366431+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:23:18.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:18 smithi088 bash[24895]: cluster 2024-09-13T00:23:17.196016+0000 mgr.y (mgr.14150) 138 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:18.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:18 smithi088 bash[24895]: audit 2024-09-13T00:23:17.354196+0000 mon.a (mon.0) 306 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/1691613782,v1:172.21.15.77:6811/1691613782]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]': finished 2024-09-13T00:23:18.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:18 smithi088 bash[24895]: cluster 2024-09-13T00:23:17.354354+0000 mon.a (mon.0) 307 : cluster [DBG] osdmap e12: 2 total, 1 up, 2 in 2024-09-13T00:23:18.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:18 smithi088 bash[24895]: audit 2024-09-13T00:23:17.355240+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:23:18.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:18 smithi088 bash[24895]: audit 2024-09-13T00:23:17.366431+0000 mon.a (mon.0) 309 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:23:19.545 INFO:teuthology.orchestra.run.smithi077.stdout:Created osd(s) 1 on host 'smithi077' 2024-09-13T00:23:19.771 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:19 smithi077 bash[22596]: cluster 2024-09-13T00:23:17.132335+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:23:19.771 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:19 smithi077 bash[22596]: cluster 2024-09-13T00:23:17.132402+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:23:19.771 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:19 smithi077 bash[22596]: cluster 2024-09-13T00:23:18.367085+0000 mon.a (mon.0) 310 : cluster [INF] osd.1 [v2:172.21.15.77:6810/1691613782,v1:172.21.15.77:6811/1691613782] boot 2024-09-13T00:23:19.771 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:19 smithi077 bash[22596]: cluster 2024-09-13T00:23:18.367166+0000 mon.a (mon.0) 311 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-09-13T00:23:19.771 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:19 smithi077 bash[22596]: audit 2024-09-13T00:23:18.368008+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:23:19.772 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:19 smithi077 bash[32040]: cluster 2024-09-13T00:23:17.132335+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:23:19.772 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:19 smithi077 bash[32040]: cluster 2024-09-13T00:23:17.132402+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:23:19.772 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:19 smithi077 bash[32040]: cluster 2024-09-13T00:23:18.367085+0000 mon.a (mon.0) 310 : cluster [INF] osd.1 [v2:172.21.15.77:6810/1691613782,v1:172.21.15.77:6811/1691613782] boot 2024-09-13T00:23:19.772 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:19 smithi077 bash[32040]: cluster 2024-09-13T00:23:18.367166+0000 mon.a (mon.0) 311 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-09-13T00:23:19.772 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:19 smithi077 bash[32040]: audit 2024-09-13T00:23:18.368008+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:23:19.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:19 smithi088 bash[24895]: cluster 2024-09-13T00:23:17.132335+0000 osd.1 (osd.1) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:23:19.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:19 smithi088 bash[24895]: cluster 2024-09-13T00:23:17.132402+0000 osd.1 (osd.1) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:23:19.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:19 smithi088 bash[24895]: cluster 2024-09-13T00:23:18.367085+0000 mon.a (mon.0) 310 : cluster [INF] osd.1 [v2:172.21.15.77:6810/1691613782,v1:172.21.15.77:6811/1691613782] boot 2024-09-13T00:23:19.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:19 smithi088 bash[24895]: cluster 2024-09-13T00:23:18.367166+0000 mon.a (mon.0) 311 : cluster [DBG] osdmap e13: 2 total, 2 up, 2 in 2024-09-13T00:23:19.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:19 smithi088 bash[24895]: audit 2024-09-13T00:23:18.368008+0000 mon.a (mon.0) 312 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:23:20.290 DEBUG:teuthology.orchestra.run.smithi077:osd.1> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.1.service 2024-09-13T00:23:20.292 INFO:tasks.cephadm:Deploying osd.2 on smithi077 with /dev/vg_nvme/lv_2... 2024-09-13T00:23:20.293 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-09-13T00:23:20.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:20 smithi077 bash[32040]: cluster 2024-09-13T00:23:19.196461+0000 mgr.y (mgr.14150) 139 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:20.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:20 smithi077 bash[32040]: cluster 2024-09-13T00:23:19.383702+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-09-13T00:23:20.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:20 smithi077 bash[32040]: audit 2024-09-13T00:23:19.526744+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:23:20.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:20 smithi077 bash[32040]: audit 2024-09-13T00:23:19.534185+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:20.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:20 smithi077 bash[32040]: audit 2024-09-13T00:23:19.542215+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:20.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:20 smithi077 bash[22596]: cluster 2024-09-13T00:23:19.196461+0000 mgr.y (mgr.14150) 139 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:20.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:20 smithi077 bash[22596]: cluster 2024-09-13T00:23:19.383702+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-09-13T00:23:20.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:20 smithi077 bash[22596]: audit 2024-09-13T00:23:19.526744+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:23:20.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:20 smithi077 bash[22596]: audit 2024-09-13T00:23:19.534185+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:20.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:20 smithi077 bash[22596]: audit 2024-09-13T00:23:19.542215+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:20.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:20 smithi088 bash[24895]: cluster 2024-09-13T00:23:19.196461+0000 mgr.y (mgr.14150) 139 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-09-13T00:23:20.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:20 smithi088 bash[24895]: cluster 2024-09-13T00:23:19.383702+0000 mon.a (mon.0) 313 : cluster [DBG] osdmap e14: 2 total, 2 up, 2 in 2024-09-13T00:23:20.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:20 smithi088 bash[24895]: audit 2024-09-13T00:23:19.526744+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:23:20.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:20 smithi088 bash[24895]: audit 2024-09-13T00:23:19.534185+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:20.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:20 smithi088 bash[24895]: audit 2024-09-13T00:23:19.542215+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:22.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:22 smithi077 bash[22596]: cluster 2024-09-13T00:23:21.196905+0000 mgr.y (mgr.14150) 140 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:22.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:22 smithi077 bash[32040]: cluster 2024-09-13T00:23:21.196905+0000 mgr.y (mgr.14150) 140 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:22.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:22 smithi088 bash[24895]: cluster 2024-09-13T00:23:21.196905+0000 mgr.y (mgr.14150) 140 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:24.300 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:23:24.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:24 smithi077 bash[22596]: cluster 2024-09-13T00:23:23.197386+0000 mgr.y (mgr.14150) 141 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:24.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:24 smithi077 bash[32040]: cluster 2024-09-13T00:23:23.197386+0000 mgr.y (mgr.14150) 141 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:24.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:24 smithi088 bash[24895]: cluster 2024-09-13T00:23:23.197386+0000 mgr.y (mgr.14150) 141 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:26.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:26 smithi077 bash[22596]: cluster 2024-09-13T00:23:25.197843+0000 mgr.y (mgr.14150) 142 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:26.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:26 smithi077 bash[32040]: cluster 2024-09-13T00:23:25.197843+0000 mgr.y (mgr.14150) 142 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:26.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:26 smithi088 bash[24895]: cluster 2024-09-13T00:23:25.197843+0000 mgr.y (mgr.14150) 142 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:28.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:28 smithi077 bash[22596]: cluster 2024-09-13T00:23:27.198285+0000 mgr.y (mgr.14150) 143 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:28.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:28 smithi077 bash[32040]: cluster 2024-09-13T00:23:27.198285+0000 mgr.y (mgr.14150) 143 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:28.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:28 smithi088 bash[24895]: cluster 2024-09-13T00:23:27.198285+0000 mgr.y (mgr.14150) 143 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:29.287 INFO:teuthology.orchestra.run.smithi077.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-09-13T00:23:29.484 INFO:teuthology.orchestra.run.smithi077.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-09-13T00:23:29.500 INFO:teuthology.orchestra.run.smithi077.stderr: stderr: 10+0 records in 2024-09-13T00:23:29.500 INFO:teuthology.orchestra.run.smithi077.stderr:10+0 records out 2024-09-13T00:23:29.500 INFO:teuthology.orchestra.run.smithi077.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0140781 s, 745 MB/s 2024-09-13T00:23:29.500 INFO:teuthology.orchestra.run.smithi077.stderr:--> Zapping successful for: 2024-09-13T00:23:30.342 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi077:vg_nvme/lv_2 2024-09-13T00:23:30.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:30 smithi077 bash[22596]: cluster 2024-09-13T00:23:29.198606+0000 mgr.y (mgr.14150) 144 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:30.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:30 smithi077 bash[32040]: cluster 2024-09-13T00:23:29.198606+0000 mgr.y (mgr.14150) 144 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:30.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:30 smithi088 bash[24895]: cluster 2024-09-13T00:23:29.198606+0000 mgr.y (mgr.14150) 144 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:32.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[32040]: cephadm 2024-09-13T00:23:30.858690+0000 mgr.y (mgr.14150) 145 : cephadm [INF] Detected new or changed devices on smithi077 2024-09-13T00:23:32.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[32040]: audit 2024-09-13T00:23:30.867786+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:32.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[32040]: audit 2024-09-13T00:23:30.876861+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:32.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[32040]: audit 2024-09-13T00:23:30.878831+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:23:32.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[32040]: audit 2024-09-13T00:23:30.880509+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:23:32.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[32040]: audit 2024-09-13T00:23:30.889910+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:32.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[32040]: audit 2024-09-13T00:23:30.892357+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:23:32.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[32040]: audit 2024-09-13T00:23:30.893930+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:23:32.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[32040]: audit 2024-09-13T00:23:30.903251+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:32.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[22596]: cephadm 2024-09-13T00:23:30.858690+0000 mgr.y (mgr.14150) 145 : cephadm [INF] Detected new or changed devices on smithi077 2024-09-13T00:23:32.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[22596]: audit 2024-09-13T00:23:30.867786+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:32.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[22596]: audit 2024-09-13T00:23:30.876861+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:32.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[22596]: audit 2024-09-13T00:23:30.878831+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:23:32.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[22596]: audit 2024-09-13T00:23:30.880509+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:23:32.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[22596]: audit 2024-09-13T00:23:30.889910+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:32.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[22596]: audit 2024-09-13T00:23:30.892357+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:23:32.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[22596]: audit 2024-09-13T00:23:30.893930+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:23:32.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:31 smithi077 bash[22596]: audit 2024-09-13T00:23:30.903251+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:32.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:31 smithi088 bash[24895]: cephadm 2024-09-13T00:23:30.858690+0000 mgr.y (mgr.14150) 145 : cephadm [INF] Detected new or changed devices on smithi077 2024-09-13T00:23:32.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:31 smithi088 bash[24895]: audit 2024-09-13T00:23:30.867786+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:32.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:31 smithi088 bash[24895]: audit 2024-09-13T00:23:30.876861+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:32.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:31 smithi088 bash[24895]: audit 2024-09-13T00:23:30.878831+0000 mon.a (mon.0) 319 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:23:32.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:31 smithi088 bash[24895]: audit 2024-09-13T00:23:30.880509+0000 mon.a (mon.0) 320 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:23:32.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:31 smithi088 bash[24895]: audit 2024-09-13T00:23:30.889910+0000 mon.a (mon.0) 321 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:32.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:31 smithi088 bash[24895]: audit 2024-09-13T00:23:30.892357+0000 mon.a (mon.0) 322 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:23:32.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:31 smithi088 bash[24895]: audit 2024-09-13T00:23:30.893930+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:23:32.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:31 smithi088 bash[24895]: audit 2024-09-13T00:23:30.903251+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:23:33.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:32 smithi077 bash[22596]: cephadm 2024-09-13T00:23:30.881643+0000 mgr.y (mgr.14150) 146 : cephadm [INF] Adjusting osd_memory_target on smithi077 to 8120M 2024-09-13T00:23:33.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:32 smithi077 bash[22596]: cluster 2024-09-13T00:23:31.199017+0000 mgr.y (mgr.14150) 147 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:33.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:32 smithi077 bash[32040]: cephadm 2024-09-13T00:23:30.881643+0000 mgr.y (mgr.14150) 146 : cephadm [INF] Adjusting osd_memory_target on smithi077 to 8120M 2024-09-13T00:23:33.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:32 smithi077 bash[32040]: cluster 2024-09-13T00:23:31.199017+0000 mgr.y (mgr.14150) 147 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:33.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:32 smithi088 bash[24895]: cephadm 2024-09-13T00:23:30.881643+0000 mgr.y (mgr.14150) 146 : cephadm [INF] Adjusting osd_memory_target on smithi077 to 8120M 2024-09-13T00:23:33.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:32 smithi088 bash[24895]: cluster 2024-09-13T00:23:31.199017+0000 mgr.y (mgr.14150) 147 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:35.082 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:23:35.149 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:34 smithi077 bash[32040]: cluster 2024-09-13T00:23:33.199337+0000 mgr.y (mgr.14150) 148 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:35.149 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:34 smithi077 bash[22596]: cluster 2024-09-13T00:23:33.199337+0000 mgr.y (mgr.14150) 148 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:35.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:34 smithi088 bash[24895]: cluster 2024-09-13T00:23:33.199337+0000 mgr.y (mgr.14150) 148 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:37.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:36 smithi077 bash[32040]: cluster 2024-09-13T00:23:35.199767+0000 mgr.y (mgr.14150) 149 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:37.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:36 smithi077 bash[22596]: cluster 2024-09-13T00:23:35.199767+0000 mgr.y (mgr.14150) 149 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:37.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:36 smithi088 bash[24895]: cluster 2024-09-13T00:23:35.199767+0000 mgr.y (mgr.14150) 149 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:38.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:37 smithi077 bash[22596]: cluster 2024-09-13T00:23:37.200165+0000 mgr.y (mgr.14150) 150 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:38.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:37 smithi077 bash[32040]: cluster 2024-09-13T00:23:37.200165+0000 mgr.y (mgr.14150) 150 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:38.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:37 smithi088 bash[24895]: cluster 2024-09-13T00:23:37.200165+0000 mgr.y (mgr.14150) 150 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:39.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:38 smithi077 bash[32040]: audit 2024-09-13T00:23:38.469126+0000 mgr.y (mgr.14150) 151 : audit [DBG] from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi077:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:23:39.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:38 smithi077 bash[32040]: audit 2024-09-13T00:23:38.472760+0000 mon.a (mon.0) 325 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:23:39.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:38 smithi077 bash[32040]: audit 2024-09-13T00:23:38.478102+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:23:39.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:38 smithi077 bash[32040]: audit 2024-09-13T00:23:38.479474+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:23:39.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:38 smithi077 bash[22596]: audit 2024-09-13T00:23:38.469126+0000 mgr.y (mgr.14150) 151 : audit [DBG] from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi077:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:23:39.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:38 smithi077 bash[22596]: audit 2024-09-13T00:23:38.472760+0000 mon.a (mon.0) 325 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:23:39.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:38 smithi077 bash[22596]: audit 2024-09-13T00:23:38.478102+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:23:39.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:38 smithi077 bash[22596]: audit 2024-09-13T00:23:38.479474+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:23:39.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:38 smithi088 bash[24895]: audit 2024-09-13T00:23:38.469126+0000 mgr.y (mgr.14150) 151 : audit [DBG] from='client.14265 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi077:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:23:39.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:38 smithi088 bash[24895]: audit 2024-09-13T00:23:38.472760+0000 mon.a (mon.0) 325 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:23:39.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:38 smithi088 bash[24895]: audit 2024-09-13T00:23:38.478102+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:23:39.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:38 smithi088 bash[24895]: audit 2024-09-13T00:23:38.479474+0000 mon.a (mon.0) 327 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:23:40.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:39 smithi077 bash[32040]: cluster 2024-09-13T00:23:39.200601+0000 mgr.y (mgr.14150) 152 : cluster [DBG] pgmap v116: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:40.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:39 smithi077 bash[22596]: cluster 2024-09-13T00:23:39.200601+0000 mgr.y (mgr.14150) 152 : cluster [DBG] pgmap v116: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:40.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:39 smithi088 bash[24895]: cluster 2024-09-13T00:23:39.200601+0000 mgr.y (mgr.14150) 152 : cluster [DBG] pgmap v116: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:42.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:42 smithi077 bash[22596]: cluster 2024-09-13T00:23:41.201053+0000 mgr.y (mgr.14150) 153 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:42.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:42 smithi077 bash[32040]: cluster 2024-09-13T00:23:41.201053+0000 mgr.y (mgr.14150) 153 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:42.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:42 smithi088 bash[24895]: cluster 2024-09-13T00:23:41.201053+0000 mgr.y (mgr.14150) 153 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:44.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:44 smithi077 bash[22596]: cluster 2024-09-13T00:23:43.201596+0000 mgr.y (mgr.14150) 154 : cluster [DBG] pgmap v118: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:44.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:44 smithi077 bash[32040]: cluster 2024-09-13T00:23:43.201596+0000 mgr.y (mgr.14150) 154 : cluster [DBG] pgmap v118: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:44.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:44 smithi088 bash[24895]: cluster 2024-09-13T00:23:43.201596+0000 mgr.y (mgr.14150) 154 : cluster [DBG] pgmap v118: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:46.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:46 smithi077 bash[22596]: cluster 2024-09-13T00:23:45.202152+0000 mgr.y (mgr.14150) 155 : cluster [DBG] pgmap v119: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:46.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:46 smithi077 bash[32040]: cluster 2024-09-13T00:23:45.202152+0000 mgr.y (mgr.14150) 155 : cluster [DBG] pgmap v119: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:46.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:46 smithi088 bash[24895]: cluster 2024-09-13T00:23:45.202152+0000 mgr.y (mgr.14150) 155 : cluster [DBG] pgmap v119: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:48.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:48 smithi077 bash[32040]: cluster 2024-09-13T00:23:47.202484+0000 mgr.y (mgr.14150) 156 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:48.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:48 smithi077 bash[32040]: audit 2024-09-13T00:23:47.291277+0000 mon.a (mon.0) 328 : audit [INF] from='client.? 172.21.15.77:0/2292398311' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff8bc88a-5b11-469f-912d-d3149036b450"}]: dispatch 2024-09-13T00:23:48.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:48 smithi077 bash[32040]: audit 2024-09-13T00:23:47.300174+0000 mon.a (mon.0) 329 : audit [INF] from='client.? 172.21.15.77:0/2292398311' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ff8bc88a-5b11-469f-912d-d3149036b450"}]': finished 2024-09-13T00:23:48.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:48 smithi077 bash[32040]: cluster 2024-09-13T00:23:47.300360+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-09-13T00:23:48.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:48 smithi077 bash[32040]: audit 2024-09-13T00:23:47.300645+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:23:48.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:48 smithi077 bash[32040]: audit 2024-09-13T00:23:47.915609+0000 mon.a (mon.0) 332 : audit [DBG] from='client.? 172.21.15.77:0/1884760091' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:23:48.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:48 smithi077 bash[22596]: cluster 2024-09-13T00:23:47.202484+0000 mgr.y (mgr.14150) 156 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:48.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:48 smithi077 bash[22596]: audit 2024-09-13T00:23:47.291277+0000 mon.a (mon.0) 328 : audit [INF] from='client.? 172.21.15.77:0/2292398311' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff8bc88a-5b11-469f-912d-d3149036b450"}]: dispatch 2024-09-13T00:23:48.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:48 smithi077 bash[22596]: audit 2024-09-13T00:23:47.300174+0000 mon.a (mon.0) 329 : audit [INF] from='client.? 172.21.15.77:0/2292398311' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ff8bc88a-5b11-469f-912d-d3149036b450"}]': finished 2024-09-13T00:23:48.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:48 smithi077 bash[22596]: cluster 2024-09-13T00:23:47.300360+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-09-13T00:23:48.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:48 smithi077 bash[22596]: audit 2024-09-13T00:23:47.300645+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:23:48.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:48 smithi077 bash[22596]: audit 2024-09-13T00:23:47.915609+0000 mon.a (mon.0) 332 : audit [DBG] from='client.? 172.21.15.77:0/1884760091' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:23:48.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:48 smithi088 bash[24895]: cluster 2024-09-13T00:23:47.202484+0000 mgr.y (mgr.14150) 156 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:48.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:48 smithi088 bash[24895]: audit 2024-09-13T00:23:47.291277+0000 mon.a (mon.0) 328 : audit [INF] from='client.? 172.21.15.77:0/2292398311' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ff8bc88a-5b11-469f-912d-d3149036b450"}]: dispatch 2024-09-13T00:23:48.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:48 smithi088 bash[24895]: audit 2024-09-13T00:23:47.300174+0000 mon.a (mon.0) 329 : audit [INF] from='client.? 172.21.15.77:0/2292398311' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ff8bc88a-5b11-469f-912d-d3149036b450"}]': finished 2024-09-13T00:23:48.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:48 smithi088 bash[24895]: cluster 2024-09-13T00:23:47.300360+0000 mon.a (mon.0) 330 : cluster [DBG] osdmap e15: 3 total, 2 up, 3 in 2024-09-13T00:23:48.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:48 smithi088 bash[24895]: audit 2024-09-13T00:23:47.300645+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:23:48.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:48 smithi088 bash[24895]: audit 2024-09-13T00:23:47.915609+0000 mon.a (mon.0) 332 : audit [DBG] from='client.? 172.21.15.77:0/1884760091' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:23:50.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:50 smithi077 bash[22596]: cluster 2024-09-13T00:23:49.202929+0000 mgr.y (mgr.14150) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:50.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:50 smithi077 bash[32040]: cluster 2024-09-13T00:23:49.202929+0000 mgr.y (mgr.14150) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:50.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:50 smithi088 bash[24895]: cluster 2024-09-13T00:23:49.202929+0000 mgr.y (mgr.14150) 157 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:52.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:52 smithi077 bash[22596]: cluster 2024-09-13T00:23:51.203500+0000 mgr.y (mgr.14150) 158 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:52.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:52 smithi077 bash[32040]: cluster 2024-09-13T00:23:51.203500+0000 mgr.y (mgr.14150) 158 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:52.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:52 smithi088 bash[24895]: cluster 2024-09-13T00:23:51.203500+0000 mgr.y (mgr.14150) 158 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:54.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:54 smithi077 bash[22596]: cluster 2024-09-13T00:23:53.204019+0000 mgr.y (mgr.14150) 159 : cluster [DBG] pgmap v124: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:54.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:54 smithi077 bash[32040]: cluster 2024-09-13T00:23:53.204019+0000 mgr.y (mgr.14150) 159 : cluster [DBG] pgmap v124: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:54.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:54 smithi088 bash[24895]: cluster 2024-09-13T00:23:53.204019+0000 mgr.y (mgr.14150) 159 : cluster [DBG] pgmap v124: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:56.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:56 smithi077 bash[22596]: cluster 2024-09-13T00:23:55.204558+0000 mgr.y (mgr.14150) 160 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:56.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:56 smithi077 bash[32040]: cluster 2024-09-13T00:23:55.204558+0000 mgr.y (mgr.14150) 160 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:56.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:56 smithi088 bash[24895]: cluster 2024-09-13T00:23:55.204558+0000 mgr.y (mgr.14150) 160 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:58.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:23:58 smithi077 bash[22596]: cluster 2024-09-13T00:23:57.204921+0000 mgr.y (mgr.14150) 161 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:58.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:23:58 smithi077 bash[32040]: cluster 2024-09-13T00:23:57.204921+0000 mgr.y (mgr.14150) 161 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:23:58.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:23:58 smithi088 bash[24895]: cluster 2024-09-13T00:23:57.204921+0000 mgr.y (mgr.14150) 161 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:00.599 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:00 smithi077 bash[22596]: cluster 2024-09-13T00:23:59.205209+0000 mgr.y (mgr.14150) 162 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:00.600 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:00 smithi077 bash[32040]: cluster 2024-09-13T00:23:59.205209+0000 mgr.y (mgr.14150) 162 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:00.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:00 smithi088 bash[24895]: cluster 2024-09-13T00:23:59.205209+0000 mgr.y (mgr.14150) 162 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:01.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:01 smithi088 bash[24895]: audit 2024-09-13T00:24:00.378265+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T00:24:01.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:01 smithi088 bash[24895]: audit 2024-09-13T00:24:00.379868+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:01.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:01 smithi088 bash[24895]: cephadm 2024-09-13T00:24:00.381113+0000 mgr.y (mgr.14150) 163 : cephadm [INF] Deploying daemon osd.2 on smithi077 2024-09-13T00:24:01.735 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:01 smithi077 bash[22596]: audit 2024-09-13T00:24:00.378265+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T00:24:01.735 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:01 smithi077 bash[22596]: audit 2024-09-13T00:24:00.379868+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:01.736 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:01 smithi077 bash[22596]: cephadm 2024-09-13T00:24:00.381113+0000 mgr.y (mgr.14150) 163 : cephadm [INF] Deploying daemon osd.2 on smithi077 2024-09-13T00:24:01.736 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:01 smithi077 bash[32040]: audit 2024-09-13T00:24:00.378265+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T00:24:01.736 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:01 smithi077 bash[32040]: audit 2024-09-13T00:24:00.379868+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:01.736 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:01 smithi077 bash[32040]: cephadm 2024-09-13T00:24:00.381113+0000 mgr.y (mgr.14150) 163 : cephadm [INF] Deploying daemon osd.2 on smithi077 2024-09-13T00:24:02.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:02 smithi077 bash[22596]: cluster 2024-09-13T00:24:01.205635+0000 mgr.y (mgr.14150) 164 : cluster [DBG] pgmap v128: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:02.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:02 smithi077 bash[32040]: cluster 2024-09-13T00:24:01.205635+0000 mgr.y (mgr.14150) 164 : cluster [DBG] pgmap v128: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:02.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:02 smithi088 bash[24895]: cluster 2024-09-13T00:24:01.205635+0000 mgr.y (mgr.14150) 164 : cluster [DBG] pgmap v128: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:04.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:04 smithi088 bash[24895]: cluster 2024-09-13T00:24:03.206085+0000 mgr.y (mgr.14150) 165 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:04.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:04 smithi077 bash[22596]: cluster 2024-09-13T00:24:03.206085+0000 mgr.y (mgr.14150) 165 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:04.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:04 smithi077 bash[32040]: cluster 2024-09-13T00:24:03.206085+0000 mgr.y (mgr.14150) 165 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:05.260 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:05 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:24:05.260 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:24:05 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:24:05.260 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:24:05 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:24:05.261 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:05 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:24:05.261 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:24:05 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:24:05.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:05 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:24:05.530 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:24:05 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:24:05.530 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:24:05 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:24:05.531 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:24:05 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:24:05.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:05 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:24:06.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:06 smithi088 bash[24895]: cluster 2024-09-13T00:24:05.206378+0000 mgr.y (mgr.14150) 166 : cluster [DBG] pgmap v130: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:06.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:06 smithi088 bash[24895]: audit 2024-09-13T00:24:05.531565+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:24:06.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:06 smithi088 bash[24895]: audit 2024-09-13T00:24:05.535661+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:06.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:06 smithi088 bash[24895]: audit 2024-09-13T00:24:05.540333+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:06.754 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:06 smithi077 bash[32040]: cluster 2024-09-13T00:24:05.206378+0000 mgr.y (mgr.14150) 166 : cluster [DBG] pgmap v130: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:06.754 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:06 smithi077 bash[32040]: audit 2024-09-13T00:24:05.531565+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:24:06.754 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:06 smithi077 bash[32040]: audit 2024-09-13T00:24:05.535661+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:06.754 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:06 smithi077 bash[32040]: audit 2024-09-13T00:24:05.540333+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:06.754 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:06 smithi077 bash[22596]: cluster 2024-09-13T00:24:05.206378+0000 mgr.y (mgr.14150) 166 : cluster [DBG] pgmap v130: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:06.754 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:06 smithi077 bash[22596]: audit 2024-09-13T00:24:05.531565+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:24:06.754 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:06 smithi077 bash[22596]: audit 2024-09-13T00:24:05.535661+0000 mon.a (mon.0) 336 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:06.755 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:06 smithi077 bash[22596]: audit 2024-09-13T00:24:05.540333+0000 mon.a (mon.0) 337 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:08.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:08 smithi077 bash[22596]: cluster 2024-09-13T00:24:07.206713+0000 mgr.y (mgr.14150) 167 : cluster [DBG] pgmap v131: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:08.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:08 smithi077 bash[32040]: cluster 2024-09-13T00:24:07.206713+0000 mgr.y (mgr.14150) 167 : cluster [DBG] pgmap v131: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:08.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:08 smithi088 bash[24895]: cluster 2024-09-13T00:24:07.206713+0000 mgr.y (mgr.14150) 167 : cluster [DBG] pgmap v131: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:10.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:10 smithi088 bash[24895]: cluster 2024-09-13T00:24:09.207005+0000 mgr.y (mgr.14150) 168 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:10.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:10 smithi077 bash[22596]: cluster 2024-09-13T00:24:09.207005+0000 mgr.y (mgr.14150) 168 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:10.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:10 smithi077 bash[32040]: cluster 2024-09-13T00:24:09.207005+0000 mgr.y (mgr.14150) 168 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:13.001 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:12 smithi077 bash[22596]: cluster 2024-09-13T00:24:11.207477+0000 mgr.y (mgr.14150) 169 : cluster [DBG] pgmap v133: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:13.001 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:12 smithi077 bash[22596]: audit 2024-09-13T00:24:11.831941+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:13.002 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:12 smithi077 bash[22596]: audit 2024-09-13T00:24:11.837983+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:13.002 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:12 smithi077 bash[22596]: audit 2024-09-13T00:24:12.190213+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:13.002 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:12 smithi077 bash[22596]: audit 2024-09-13T00:24:12.191376+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:24:13.002 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:12 smithi077 bash[22596]: audit 2024-09-13T00:24:12.199612+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:13.002 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:12 smithi077 bash[22596]: audit 2024-09-13T00:24:12.749850+0000 mon.c (mon.2) 4 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/2379210664,v1:172.21.15.77:6819/2379210664]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T00:24:13.002 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:12 smithi077 bash[22596]: audit 2024-09-13T00:24:12.750223+0000 mon.a (mon.0) 343 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T00:24:13.002 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:12 smithi077 bash[32040]: cluster 2024-09-13T00:24:11.207477+0000 mgr.y (mgr.14150) 169 : cluster [DBG] pgmap v133: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:13.002 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:12 smithi077 bash[32040]: audit 2024-09-13T00:24:11.831941+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:13.002 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:12 smithi077 bash[32040]: audit 2024-09-13T00:24:11.837983+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:13.002 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:12 smithi077 bash[32040]: audit 2024-09-13T00:24:12.190213+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:13.003 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:12 smithi077 bash[32040]: audit 2024-09-13T00:24:12.191376+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:24:13.003 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:12 smithi077 bash[32040]: audit 2024-09-13T00:24:12.199612+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:13.003 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:12 smithi077 bash[32040]: audit 2024-09-13T00:24:12.749850+0000 mon.c (mon.2) 4 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/2379210664,v1:172.21.15.77:6819/2379210664]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T00:24:13.003 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:12 smithi077 bash[32040]: audit 2024-09-13T00:24:12.750223+0000 mon.a (mon.0) 343 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T00:24:13.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:12 smithi088 bash[24895]: cluster 2024-09-13T00:24:11.207477+0000 mgr.y (mgr.14150) 169 : cluster [DBG] pgmap v133: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:13.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:12 smithi088 bash[24895]: audit 2024-09-13T00:24:11.831941+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:13.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:12 smithi088 bash[24895]: audit 2024-09-13T00:24:11.837983+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:13.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:12 smithi088 bash[24895]: audit 2024-09-13T00:24:12.190213+0000 mon.a (mon.0) 340 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:13.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:12 smithi088 bash[24895]: audit 2024-09-13T00:24:12.191376+0000 mon.a (mon.0) 341 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:24:13.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:12 smithi088 bash[24895]: audit 2024-09-13T00:24:12.199612+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:13.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:12 smithi088 bash[24895]: audit 2024-09-13T00:24:12.749850+0000 mon.c (mon.2) 4 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/2379210664,v1:172.21.15.77:6819/2379210664]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T00:24:13.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:12 smithi088 bash[24895]: audit 2024-09-13T00:24:12.750223+0000 mon.a (mon.0) 343 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T00:24:14.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:14 smithi077 bash[32040]: audit 2024-09-13T00:24:13.205171+0000 mon.a (mon.0) 344 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T00:24:14.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:14 smithi077 bash[32040]: cluster 2024-09-13T00:24:13.205287+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-09-13T00:24:14.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:14 smithi077 bash[32040]: audit 2024-09-13T00:24:13.205491+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:24:14.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:14 smithi077 bash[32040]: audit 2024-09-13T00:24:13.206643+0000 mon.c (mon.2) 5 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/2379210664,v1:172.21.15.77:6819/2379210664]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:24:14.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:14 smithi077 bash[32040]: audit 2024-09-13T00:24:13.207386+0000 mon.a (mon.0) 347 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:24:14.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:14 smithi077 bash[32040]: cluster 2024-09-13T00:24:13.207873+0000 mgr.y (mgr.14150) 170 : cluster [DBG] pgmap v135: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:14.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:14 smithi077 bash[22596]: audit 2024-09-13T00:24:13.205171+0000 mon.a (mon.0) 344 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T00:24:14.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:14 smithi077 bash[22596]: cluster 2024-09-13T00:24:13.205287+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-09-13T00:24:14.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:14 smithi077 bash[22596]: audit 2024-09-13T00:24:13.205491+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:24:14.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:14 smithi077 bash[22596]: audit 2024-09-13T00:24:13.206643+0000 mon.c (mon.2) 5 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/2379210664,v1:172.21.15.77:6819/2379210664]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:24:14.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:14 smithi077 bash[22596]: audit 2024-09-13T00:24:13.207386+0000 mon.a (mon.0) 347 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:24:14.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:14 smithi077 bash[22596]: cluster 2024-09-13T00:24:13.207873+0000 mgr.y (mgr.14150) 170 : cluster [DBG] pgmap v135: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:14.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:14 smithi088 bash[24895]: audit 2024-09-13T00:24:13.205171+0000 mon.a (mon.0) 344 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T00:24:14.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:14 smithi088 bash[24895]: cluster 2024-09-13T00:24:13.205287+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e16: 3 total, 2 up, 3 in 2024-09-13T00:24:14.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:14 smithi088 bash[24895]: audit 2024-09-13T00:24:13.205491+0000 mon.a (mon.0) 346 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:24:14.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:14 smithi088 bash[24895]: audit 2024-09-13T00:24:13.206643+0000 mon.c (mon.2) 5 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/2379210664,v1:172.21.15.77:6819/2379210664]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:24:14.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:14 smithi088 bash[24895]: audit 2024-09-13T00:24:13.207386+0000 mon.a (mon.0) 347 : audit [INF] from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:24:14.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:14 smithi088 bash[24895]: cluster 2024-09-13T00:24:13.207873+0000 mgr.y (mgr.14150) 170 : cluster [DBG] pgmap v135: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:15.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:15 smithi077 bash[22596]: audit 2024-09-13T00:24:14.207481+0000 mon.a (mon.0) 348 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]': finished 2024-09-13T00:24:15.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:15 smithi077 bash[22596]: cluster 2024-09-13T00:24:14.207668+0000 mon.a (mon.0) 349 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-09-13T00:24:15.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:15 smithi077 bash[22596]: audit 2024-09-13T00:24:14.208783+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:24:15.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:15 smithi077 bash[22596]: audit 2024-09-13T00:24:14.218792+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:24:15.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:15 smithi077 bash[32040]: audit 2024-09-13T00:24:14.207481+0000 mon.a (mon.0) 348 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]': finished 2024-09-13T00:24:15.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:15 smithi077 bash[32040]: cluster 2024-09-13T00:24:14.207668+0000 mon.a (mon.0) 349 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-09-13T00:24:15.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:15 smithi077 bash[32040]: audit 2024-09-13T00:24:14.208783+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:24:15.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:15 smithi077 bash[32040]: audit 2024-09-13T00:24:14.218792+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:24:15.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:15 smithi088 bash[24895]: audit 2024-09-13T00:24:14.207481+0000 mon.a (mon.0) 348 : audit [INF] from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]': finished 2024-09-13T00:24:15.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:15 smithi088 bash[24895]: cluster 2024-09-13T00:24:14.207668+0000 mon.a (mon.0) 349 : cluster [DBG] osdmap e17: 3 total, 2 up, 3 in 2024-09-13T00:24:15.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:15 smithi088 bash[24895]: audit 2024-09-13T00:24:14.208783+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:24:15.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:15 smithi088 bash[24895]: audit 2024-09-13T00:24:14.218792+0000 mon.a (mon.0) 351 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:24:16.221 INFO:teuthology.orchestra.run.smithi077.stdout:Created osd(s) 2 on host 'smithi077' 2024-09-13T00:24:16.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[22596]: cluster 2024-09-13T00:24:13.725917+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:24:16.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[22596]: cluster 2024-09-13T00:24:13.726004+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:24:16.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[22596]: cluster 2024-09-13T00:24:15.208361+0000 mgr.y (mgr.14150) 171 : cluster [DBG] pgmap v137: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:16.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[22596]: cluster 2024-09-13T00:24:15.217898+0000 mon.a (mon.0) 352 : cluster [INF] osd.2 [v2:172.21.15.77:6818/2379210664,v1:172.21.15.77:6819/2379210664] boot 2024-09-13T00:24:16.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[22596]: cluster 2024-09-13T00:24:15.217969+0000 mon.a (mon.0) 353 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-09-13T00:24:16.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[22596]: audit 2024-09-13T00:24:15.218601+0000 mon.a (mon.0) 354 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:24:16.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[22596]: audit 2024-09-13T00:24:15.321491+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T00:24:16.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[22596]: audit 2024-09-13T00:24:16.197146+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:24:16.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[22596]: audit 2024-09-13T00:24:16.205825+0000 mon.a (mon.0) 357 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:16.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[22596]: audit 2024-09-13T00:24:16.214749+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:16.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[32040]: cluster 2024-09-13T00:24:13.725917+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:24:16.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[32040]: cluster 2024-09-13T00:24:13.726004+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:24:16.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[32040]: cluster 2024-09-13T00:24:15.208361+0000 mgr.y (mgr.14150) 171 : cluster [DBG] pgmap v137: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:16.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[32040]: cluster 2024-09-13T00:24:15.217898+0000 mon.a (mon.0) 352 : cluster [INF] osd.2 [v2:172.21.15.77:6818/2379210664,v1:172.21.15.77:6819/2379210664] boot 2024-09-13T00:24:16.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[32040]: cluster 2024-09-13T00:24:15.217969+0000 mon.a (mon.0) 353 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-09-13T00:24:16.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[32040]: audit 2024-09-13T00:24:15.218601+0000 mon.a (mon.0) 354 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:24:16.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[32040]: audit 2024-09-13T00:24:15.321491+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T00:24:16.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[32040]: audit 2024-09-13T00:24:16.197146+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:24:16.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[32040]: audit 2024-09-13T00:24:16.205825+0000 mon.a (mon.0) 357 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:16.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:16 smithi077 bash[32040]: audit 2024-09-13T00:24:16.214749+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:16.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:16 smithi088 bash[24895]: cluster 2024-09-13T00:24:13.725917+0000 osd.2 (osd.2) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:24:16.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:16 smithi088 bash[24895]: cluster 2024-09-13T00:24:13.726004+0000 osd.2 (osd.2) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:24:16.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:16 smithi088 bash[24895]: cluster 2024-09-13T00:24:15.208361+0000 mgr.y (mgr.14150) 171 : cluster [DBG] pgmap v137: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-09-13T00:24:16.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:16 smithi088 bash[24895]: cluster 2024-09-13T00:24:15.217898+0000 mon.a (mon.0) 352 : cluster [INF] osd.2 [v2:172.21.15.77:6818/2379210664,v1:172.21.15.77:6819/2379210664] boot 2024-09-13T00:24:16.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:16 smithi088 bash[24895]: cluster 2024-09-13T00:24:15.217969+0000 mon.a (mon.0) 353 : cluster [DBG] osdmap e18: 3 total, 3 up, 3 in 2024-09-13T00:24:16.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:16 smithi088 bash[24895]: audit 2024-09-13T00:24:15.218601+0000 mon.a (mon.0) 354 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:24:16.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:16 smithi088 bash[24895]: audit 2024-09-13T00:24:15.321491+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-09-13T00:24:16.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:16 smithi088 bash[24895]: audit 2024-09-13T00:24:16.197146+0000 mon.a (mon.0) 356 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:24:16.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:16 smithi088 bash[24895]: audit 2024-09-13T00:24:16.205825+0000 mon.a (mon.0) 357 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:16.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:16 smithi088 bash[24895]: audit 2024-09-13T00:24:16.214749+0000 mon.a (mon.0) 358 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:17.000 DEBUG:teuthology.orchestra.run.smithi077:osd.2> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.2.service 2024-09-13T00:24:17.003 INFO:tasks.cephadm:Deploying osd.3 on smithi077 with /dev/vg_nvme/lv_1... 2024-09-13T00:24:17.003 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-09-13T00:24:17.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:17 smithi077 bash[32040]: audit 2024-09-13T00:24:16.236213+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T00:24:17.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:17 smithi077 bash[32040]: cluster 2024-09-13T00:24:16.236286+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-09-13T00:24:17.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:17 smithi077 bash[32040]: audit 2024-09-13T00:24:16.237893+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:24:17.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:17 smithi077 bash[22596]: audit 2024-09-13T00:24:16.236213+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T00:24:17.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:17 smithi077 bash[22596]: cluster 2024-09-13T00:24:16.236286+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-09-13T00:24:17.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:17 smithi077 bash[22596]: audit 2024-09-13T00:24:16.237893+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:24:17.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:17 smithi088 bash[24895]: audit 2024-09-13T00:24:16.236213+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-09-13T00:24:17.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:17 smithi088 bash[24895]: cluster 2024-09-13T00:24:16.236286+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e19: 3 total, 3 up, 3 in 2024-09-13T00:24:17.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:17 smithi088 bash[24895]: audit 2024-09-13T00:24:16.237893+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:24:18.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:18 smithi077 bash[22596]: cluster 2024-09-13T00:24:17.208908+0000 mgr.y (mgr.14150) 172 : cluster [DBG] pgmap v140: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:18.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:18 smithi077 bash[22596]: cluster 2024-09-13T00:24:17.232544+0000 mon.a (mon.0) 362 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:24:18.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:18 smithi077 bash[22596]: audit 2024-09-13T00:24:17.241706+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T00:24:18.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:18 smithi077 bash[22596]: cluster 2024-09-13T00:24:17.241798+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-09-13T00:24:18.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:18 smithi077 bash[22596]: audit 2024-09-13T00:24:18.156441+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:18.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:18 smithi077 bash[32040]: cluster 2024-09-13T00:24:17.208908+0000 mgr.y (mgr.14150) 172 : cluster [DBG] pgmap v140: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:18.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:18 smithi077 bash[32040]: cluster 2024-09-13T00:24:17.232544+0000 mon.a (mon.0) 362 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:24:18.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:18 smithi077 bash[32040]: audit 2024-09-13T00:24:17.241706+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T00:24:18.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:18 smithi077 bash[32040]: cluster 2024-09-13T00:24:17.241798+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-09-13T00:24:18.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:18 smithi077 bash[32040]: audit 2024-09-13T00:24:18.156441+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:18.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:18 smithi088 bash[24895]: cluster 2024-09-13T00:24:17.208908+0000 mgr.y (mgr.14150) 172 : cluster [DBG] pgmap v140: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:18.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:18 smithi088 bash[24895]: cluster 2024-09-13T00:24:17.232544+0000 mon.a (mon.0) 362 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:24:18.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:18 smithi088 bash[24895]: audit 2024-09-13T00:24:17.241706+0000 mon.a (mon.0) 363 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-13T00:24:18.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:18 smithi088 bash[24895]: cluster 2024-09-13T00:24:17.241798+0000 mon.a (mon.0) 364 : cluster [DBG] osdmap e20: 3 total, 3 up, 3 in 2024-09-13T00:24:18.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:18 smithi088 bash[24895]: audit 2024-09-13T00:24:18.156441+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:19.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:19 smithi077 bash[22596]: cluster 2024-09-13T00:24:18.264911+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-09-13T00:24:19.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:19 smithi077 bash[22596]: cluster 2024-09-13T00:24:18.265853+0000 mon.a (mon.0) 367 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2024-09-13T00:24:19.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:19 smithi077 bash[32040]: cluster 2024-09-13T00:24:18.264911+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-09-13T00:24:19.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:19 smithi077 bash[32040]: cluster 2024-09-13T00:24:18.265853+0000 mon.a (mon.0) 367 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2024-09-13T00:24:19.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:19 smithi088 bash[24895]: cluster 2024-09-13T00:24:18.264911+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e21: 3 total, 3 up, 3 in 2024-09-13T00:24:19.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:19 smithi088 bash[24895]: cluster 2024-09-13T00:24:18.265853+0000 mon.a (mon.0) 367 : cluster [DBG] mgrmap e15: y(active, since 4m), standbys: x 2024-09-13T00:24:20.993 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:24:21.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:20 smithi077 bash[22596]: cluster 2024-09-13T00:24:19.251813+0000 mon.a (mon.0) 368 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:24:21.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:20 smithi077 bash[22596]: cluster 2024-09-13T00:24:19.251852+0000 mon.a (mon.0) 369 : cluster [INF] Cluster is now healthy 2024-09-13T00:24:21.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:20 smithi077 bash[22596]: audit 2024-09-13T00:24:19.880910+0000 mon.a (mon.0) 370 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:24:21.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:20 smithi077 bash[32040]: cluster 2024-09-13T00:24:19.251813+0000 mon.a (mon.0) 368 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:24:21.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:20 smithi077 bash[32040]: cluster 2024-09-13T00:24:19.251852+0000 mon.a (mon.0) 369 : cluster [INF] Cluster is now healthy 2024-09-13T00:24:21.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:20 smithi077 bash[32040]: audit 2024-09-13T00:24:19.880910+0000 mon.a (mon.0) 370 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:24:21.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:20 smithi088 bash[24895]: cluster 2024-09-13T00:24:19.251813+0000 mon.a (mon.0) 368 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:24:21.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:20 smithi088 bash[24895]: cluster 2024-09-13T00:24:19.251852+0000 mon.a (mon.0) 369 : cluster [INF] Cluster is now healthy 2024-09-13T00:24:21.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:20 smithi088 bash[24895]: audit 2024-09-13T00:24:19.880910+0000 mon.a (mon.0) 370 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:24:22.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:22 smithi088 bash[24895]: cluster 2024-09-13T00:24:19.209403+0000 mgr.y (mgr.14150) 173 : cluster [DBG] pgmap v143: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:22.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:22 smithi088 bash[24895]: audit 2024-09-13T00:24:20.751034+0000 mon.a (mon.0) 371 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:24:22.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:22 smithi088 bash[24895]: audit 2024-09-13T00:24:20.753758+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:24:22.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:22 smithi088 bash[24895]: audit 2024-09-13T00:24:20.760202+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:24:22.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:22 smithi088 bash[24895]: audit 2024-09-13T00:24:20.760419+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:24:22.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:22 smithi088 bash[24895]: audit 2024-09-13T00:24:20.762960+0000 mon.b (mon.1) 7 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:24:22.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:22 smithi088 bash[24895]: audit 2024-09-13T00:24:20.763124+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:24:22.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:22 smithi088 bash[24895]: audit 2024-09-13T00:24:20.763309+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:24:22.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:22 smithi088 bash[24895]: audit 2024-09-13T00:24:20.763434+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:24:22.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:22 smithi088 bash[24895]: audit 2024-09-13T00:24:21.658714+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:24:22.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:22 smithi088 bash[24895]: audit 2024-09-13T00:24:21.711376+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:22.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:22 smithi088 bash[24895]: audit 2024-09-13T00:24:21.714341+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:24:22.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:22 smithi088 bash[24895]: audit 2024-09-13T00:24:21.714661+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:24:22.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:22 smithi088 bash[24895]: audit 2024-09-13T00:24:21.714898+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:24:22.974 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[22596]: cluster 2024-09-13T00:24:19.209403+0000 mgr.y (mgr.14150) 173 : cluster [DBG] pgmap v143: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:22.974 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[22596]: audit 2024-09-13T00:24:20.751034+0000 mon.a (mon.0) 371 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:24:22.975 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[22596]: audit 2024-09-13T00:24:20.753758+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:24:22.975 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[22596]: audit 2024-09-13T00:24:20.760202+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:24:22.975 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[22596]: audit 2024-09-13T00:24:20.760419+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:24:22.975 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[22596]: audit 2024-09-13T00:24:20.762960+0000 mon.b (mon.1) 7 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:24:22.975 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[22596]: audit 2024-09-13T00:24:20.763124+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:24:22.975 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[22596]: audit 2024-09-13T00:24:20.763309+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:24:22.975 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[22596]: audit 2024-09-13T00:24:20.763434+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:24:22.975 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[22596]: audit 2024-09-13T00:24:21.658714+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:24:22.975 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[22596]: audit 2024-09-13T00:24:21.711376+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:22.976 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[22596]: audit 2024-09-13T00:24:21.714341+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:24:22.976 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[22596]: audit 2024-09-13T00:24:21.714661+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:24:22.976 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[22596]: audit 2024-09-13T00:24:21.714898+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:24:22.976 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[32040]: cluster 2024-09-13T00:24:19.209403+0000 mgr.y (mgr.14150) 173 : cluster [DBG] pgmap v143: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:22.976 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[32040]: audit 2024-09-13T00:24:20.751034+0000 mon.a (mon.0) 371 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:24:22.976 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[32040]: audit 2024-09-13T00:24:20.753758+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:24:22.976 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[32040]: audit 2024-09-13T00:24:20.760202+0000 mon.a (mon.0) 373 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:24:22.976 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[32040]: audit 2024-09-13T00:24:20.760419+0000 mon.a (mon.0) 374 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:24:22.976 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[32040]: audit 2024-09-13T00:24:20.762960+0000 mon.b (mon.1) 7 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:24:22.976 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[32040]: audit 2024-09-13T00:24:20.763124+0000 mon.a (mon.0) 375 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:24:22.977 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[32040]: audit 2024-09-13T00:24:20.763309+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:24:22.977 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[32040]: audit 2024-09-13T00:24:20.763434+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:24:22.977 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[32040]: audit 2024-09-13T00:24:21.658714+0000 mon.b (mon.1) 8 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:24:22.977 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[32040]: audit 2024-09-13T00:24:21.711376+0000 mon.a (mon.0) 378 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:22.977 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[32040]: audit 2024-09-13T00:24:21.714341+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:24:22.977 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[32040]: audit 2024-09-13T00:24:21.714661+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:24:22.977 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:22 smithi077 bash[32040]: audit 2024-09-13T00:24:21.714898+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:24:23.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:23 smithi077 bash[22596]: cluster 2024-09-13T00:24:21.209995+0000 mgr.y (mgr.14150) 174 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:23.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:23 smithi077 bash[22596]: audit 2024-09-13T00:24:21.713948+0000 mon.c (mon.2) 6 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:24:23.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:23 smithi077 bash[22596]: audit 2024-09-13T00:24:22.590138+0000 mon.c (mon.2) 7 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:24:23.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:23 smithi077 bash[32040]: cluster 2024-09-13T00:24:21.209995+0000 mgr.y (mgr.14150) 174 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:23.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:23 smithi077 bash[32040]: audit 2024-09-13T00:24:21.713948+0000 mon.c (mon.2) 6 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:24:23.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:23 smithi077 bash[32040]: audit 2024-09-13T00:24:22.590138+0000 mon.c (mon.2) 7 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:24:24.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:23 smithi088 bash[24895]: cluster 2024-09-13T00:24:21.209995+0000 mgr.y (mgr.14150) 174 : cluster [DBG] pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:24.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:23 smithi088 bash[24895]: audit 2024-09-13T00:24:21.713948+0000 mon.c (mon.2) 6 : audit [INF] from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-13T00:24:24.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:23 smithi088 bash[24895]: audit 2024-09-13T00:24:22.590138+0000 mon.c (mon.2) 7 : audit [INF] from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-13T00:24:25.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:24 smithi077 bash[22596]: cluster 2024-09-13T00:24:23.210541+0000 mgr.y (mgr.14150) 175 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:25.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:24 smithi077 bash[32040]: cluster 2024-09-13T00:24:23.210541+0000 mgr.y (mgr.14150) 175 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:25.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:24 smithi088 bash[24895]: cluster 2024-09-13T00:24:23.210541+0000 mgr.y (mgr.14150) 175 : cluster [DBG] pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:26.671 INFO:teuthology.orchestra.run.smithi077.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-09-13T00:24:26.849 INFO:teuthology.orchestra.run.smithi077.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-09-13T00:24:26.864 INFO:teuthology.orchestra.run.smithi077.stderr: stderr: 10+0 records in 2024-09-13T00:24:26.865 INFO:teuthology.orchestra.run.smithi077.stderr:10+0 records out 2024-09-13T00:24:26.865 INFO:teuthology.orchestra.run.smithi077.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0135284 s, 775 MB/s 2024-09-13T00:24:26.865 INFO:teuthology.orchestra.run.smithi077.stderr:--> Zapping successful for: 2024-09-13T00:24:26.885 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:26 smithi077 bash[22596]: cluster 2024-09-13T00:24:25.211090+0000 mgr.y (mgr.14150) 176 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-13T00:24:26.885 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:26 smithi077 bash[32040]: cluster 2024-09-13T00:24:25.211090+0000 mgr.y (mgr.14150) 176 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-13T00:24:27.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:26 smithi088 bash[24895]: cluster 2024-09-13T00:24:25.211090+0000 mgr.y (mgr.14150) 176 : cluster [DBG] pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-09-13T00:24:27.905 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi077:vg_nvme/lv_1 2024-09-13T00:24:29.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[22596]: cluster 2024-09-13T00:24:27.211393+0000 mgr.y (mgr.14150) 177 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-13T00:24:29.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[22596]: audit 2024-09-13T00:24:28.355078+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:29.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[22596]: audit 2024-09-13T00:24:28.364274+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:29.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[22596]: audit 2024-09-13T00:24:28.366593+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:24:29.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[22596]: audit 2024-09-13T00:24:28.368315+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:24:29.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[22596]: audit 2024-09-13T00:24:28.369924+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:24:29.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[22596]: audit 2024-09-13T00:24:28.379781+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:29.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[22596]: audit 2024-09-13T00:24:28.381856+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:29.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[22596]: audit 2024-09-13T00:24:28.383488+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:24:29.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[22596]: audit 2024-09-13T00:24:28.393254+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:29.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[32040]: cluster 2024-09-13T00:24:27.211393+0000 mgr.y (mgr.14150) 177 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-13T00:24:29.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[32040]: audit 2024-09-13T00:24:28.355078+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:29.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[32040]: audit 2024-09-13T00:24:28.364274+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:29.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[32040]: audit 2024-09-13T00:24:28.366593+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:24:29.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[32040]: audit 2024-09-13T00:24:28.368315+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:24:29.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[32040]: audit 2024-09-13T00:24:28.369924+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:24:29.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[32040]: audit 2024-09-13T00:24:28.379781+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:29.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[32040]: audit 2024-09-13T00:24:28.381856+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:29.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[32040]: audit 2024-09-13T00:24:28.383488+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:24:29.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:28 smithi077 bash[32040]: audit 2024-09-13T00:24:28.393254+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:28 smithi088 bash[24895]: cluster 2024-09-13T00:24:27.211393+0000 mgr.y (mgr.14150) 177 : cluster [DBG] pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-09-13T00:24:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:28 smithi088 bash[24895]: audit 2024-09-13T00:24:28.355078+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:28 smithi088 bash[24895]: audit 2024-09-13T00:24:28.364274+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:28 smithi088 bash[24895]: audit 2024-09-13T00:24:28.366593+0000 mon.a (mon.0) 384 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:24:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:28 smithi088 bash[24895]: audit 2024-09-13T00:24:28.368315+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:24:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:28 smithi088 bash[24895]: audit 2024-09-13T00:24:28.369924+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:24:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:28 smithi088 bash[24895]: audit 2024-09-13T00:24:28.379781+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:28 smithi088 bash[24895]: audit 2024-09-13T00:24:28.381856+0000 mon.a (mon.0) 388 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:28 smithi088 bash[24895]: audit 2024-09-13T00:24:28.383488+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:24:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:28 smithi088 bash[24895]: audit 2024-09-13T00:24:28.393254+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:24:30.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:29 smithi077 bash[22596]: cephadm 2024-09-13T00:24:28.346852+0000 mgr.y (mgr.14150) 178 : cephadm [INF] Detected new or changed devices on smithi077 2024-09-13T00:24:30.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:29 smithi077 bash[22596]: cephadm 2024-09-13T00:24:28.371138+0000 mgr.y (mgr.14150) 179 : cephadm [INF] Adjusting osd_memory_target on smithi077 to 5413M 2024-09-13T00:24:30.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:29 smithi077 bash[32040]: cephadm 2024-09-13T00:24:28.346852+0000 mgr.y (mgr.14150) 178 : cephadm [INF] Detected new or changed devices on smithi077 2024-09-13T00:24:30.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:29 smithi077 bash[32040]: cephadm 2024-09-13T00:24:28.371138+0000 mgr.y (mgr.14150) 179 : cephadm [INF] Adjusting osd_memory_target on smithi077 to 5413M 2024-09-13T00:24:30.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:29 smithi088 bash[24895]: cephadm 2024-09-13T00:24:28.346852+0000 mgr.y (mgr.14150) 178 : cephadm [INF] Detected new or changed devices on smithi077 2024-09-13T00:24:30.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:29 smithi088 bash[24895]: cephadm 2024-09-13T00:24:28.371138+0000 mgr.y (mgr.14150) 179 : cephadm [INF] Adjusting osd_memory_target on smithi077 to 5413M 2024-09-13T00:24:31.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:30 smithi077 bash[22596]: cluster 2024-09-13T00:24:29.211874+0000 mgr.y (mgr.14150) 180 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-13T00:24:31.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:30 smithi077 bash[32040]: cluster 2024-09-13T00:24:29.211874+0000 mgr.y (mgr.14150) 180 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-13T00:24:31.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:30 smithi088 bash[24895]: cluster 2024-09-13T00:24:29.211874+0000 mgr.y (mgr.14150) 180 : cluster [DBG] pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-09-13T00:24:32.654 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:24:33.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:32 smithi077 bash[22596]: cluster 2024-09-13T00:24:31.212415+0000 mgr.y (mgr.14150) 181 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-13T00:24:33.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:32 smithi077 bash[32040]: cluster 2024-09-13T00:24:31.212415+0000 mgr.y (mgr.14150) 181 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-13T00:24:33.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:32 smithi088 bash[24895]: cluster 2024-09-13T00:24:31.212415+0000 mgr.y (mgr.14150) 181 : cluster [DBG] pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-13T00:24:35.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:34 smithi077 bash[32040]: cluster 2024-09-13T00:24:33.213020+0000 mgr.y (mgr.14150) 182 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T00:24:35.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:34 smithi077 bash[22596]: cluster 2024-09-13T00:24:33.213020+0000 mgr.y (mgr.14150) 182 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T00:24:35.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:34 smithi088 bash[24895]: cluster 2024-09-13T00:24:33.213020+0000 mgr.y (mgr.14150) 182 : cluster [DBG] pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T00:24:37.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:36 smithi077 bash[22596]: cluster 2024-09-13T00:24:35.213689+0000 mgr.y (mgr.14150) 183 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T00:24:37.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:36 smithi077 bash[22596]: audit 2024-09-13T00:24:36.170180+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:24:37.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:36 smithi077 bash[22596]: audit 2024-09-13T00:24:36.175518+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:24:37.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:36 smithi077 bash[22596]: audit 2024-09-13T00:24:36.176689+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:37.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:36 smithi077 bash[32040]: cluster 2024-09-13T00:24:35.213689+0000 mgr.y (mgr.14150) 183 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T00:24:37.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:36 smithi077 bash[32040]: audit 2024-09-13T00:24:36.170180+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:24:37.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:36 smithi077 bash[32040]: audit 2024-09-13T00:24:36.175518+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:24:37.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:36 smithi077 bash[32040]: audit 2024-09-13T00:24:36.176689+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:37.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:36 smithi088 bash[24895]: cluster 2024-09-13T00:24:35.213689+0000 mgr.y (mgr.14150) 183 : cluster [DBG] pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-13T00:24:37.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:36 smithi088 bash[24895]: audit 2024-09-13T00:24:36.170180+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:24:37.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:36 smithi088 bash[24895]: audit 2024-09-13T00:24:36.175518+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:24:37.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:36 smithi088 bash[24895]: audit 2024-09-13T00:24:36.176689+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:38.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:37 smithi077 bash[22596]: audit 2024-09-13T00:24:36.164010+0000 mgr.y (mgr.14150) 184 : audit [DBG] from='client.14301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi077:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:24:38.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:37 smithi077 bash[32040]: audit 2024-09-13T00:24:36.164010+0000 mgr.y (mgr.14150) 184 : audit [DBG] from='client.14301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi077:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:24:38.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:37 smithi088 bash[24895]: audit 2024-09-13T00:24:36.164010+0000 mgr.y (mgr.14150) 184 : audit [DBG] from='client.14301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi077:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:24:39.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:38 smithi077 bash[22596]: cluster 2024-09-13T00:24:37.214267+0000 mgr.y (mgr.14150) 185 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:39.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:38 smithi077 bash[32040]: cluster 2024-09-13T00:24:37.214267+0000 mgr.y (mgr.14150) 185 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:39.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:38 smithi088 bash[24895]: cluster 2024-09-13T00:24:37.214267+0000 mgr.y (mgr.14150) 185 : cluster [DBG] pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:41.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:40 smithi077 bash[22596]: cluster 2024-09-13T00:24:39.214855+0000 mgr.y (mgr.14150) 186 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:41.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:40 smithi077 bash[32040]: cluster 2024-09-13T00:24:39.214855+0000 mgr.y (mgr.14150) 186 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:41.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:40 smithi088 bash[24895]: cluster 2024-09-13T00:24:39.214855+0000 mgr.y (mgr.14150) 186 : cluster [DBG] pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:43.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:42 smithi077 bash[22596]: cluster 2024-09-13T00:24:41.215384+0000 mgr.y (mgr.14150) 187 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:43.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:42 smithi077 bash[32040]: cluster 2024-09-13T00:24:41.215384+0000 mgr.y (mgr.14150) 187 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:43.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:42 smithi088 bash[24895]: cluster 2024-09-13T00:24:41.215384+0000 mgr.y (mgr.14150) 187 : cluster [DBG] pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:45.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:44 smithi077 bash[22596]: cluster 2024-09-13T00:24:43.215917+0000 mgr.y (mgr.14150) 188 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:45.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:44 smithi077 bash[32040]: cluster 2024-09-13T00:24:43.215917+0000 mgr.y (mgr.14150) 188 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:45.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:44 smithi088 bash[24895]: cluster 2024-09-13T00:24:43.215917+0000 mgr.y (mgr.14150) 188 : cluster [DBG] pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:46.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:45 smithi077 bash[22596]: audit 2024-09-13T00:24:45.066900+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.77:0/3934927065' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb032137-609e-41f6-bfc7-e70b77ea9456"}]: dispatch 2024-09-13T00:24:46.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:45 smithi077 bash[22596]: audit 2024-09-13T00:24:45.067483+0000 mon.a (mon.0) 394 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb032137-609e-41f6-bfc7-e70b77ea9456"}]: dispatch 2024-09-13T00:24:46.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:45 smithi077 bash[22596]: audit 2024-09-13T00:24:45.075203+0000 mon.a (mon.0) 395 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb032137-609e-41f6-bfc7-e70b77ea9456"}]': finished 2024-09-13T00:24:46.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:45 smithi077 bash[22596]: cluster 2024-09-13T00:24:45.075301+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2024-09-13T00:24:46.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:45 smithi077 bash[22596]: audit 2024-09-13T00:24:45.075530+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:24:46.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:45 smithi077 bash[32040]: audit 2024-09-13T00:24:45.066900+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.77:0/3934927065' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb032137-609e-41f6-bfc7-e70b77ea9456"}]: dispatch 2024-09-13T00:24:46.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:45 smithi077 bash[32040]: audit 2024-09-13T00:24:45.067483+0000 mon.a (mon.0) 394 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb032137-609e-41f6-bfc7-e70b77ea9456"}]: dispatch 2024-09-13T00:24:46.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:45 smithi077 bash[32040]: audit 2024-09-13T00:24:45.075203+0000 mon.a (mon.0) 395 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb032137-609e-41f6-bfc7-e70b77ea9456"}]': finished 2024-09-13T00:24:46.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:45 smithi077 bash[32040]: cluster 2024-09-13T00:24:45.075301+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2024-09-13T00:24:46.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:45 smithi077 bash[32040]: audit 2024-09-13T00:24:45.075530+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:24:46.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:45 smithi088 bash[24895]: audit 2024-09-13T00:24:45.066900+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.77:0/3934927065' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb032137-609e-41f6-bfc7-e70b77ea9456"}]: dispatch 2024-09-13T00:24:46.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:45 smithi088 bash[24895]: audit 2024-09-13T00:24:45.067483+0000 mon.a (mon.0) 394 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cb032137-609e-41f6-bfc7-e70b77ea9456"}]: dispatch 2024-09-13T00:24:46.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:45 smithi088 bash[24895]: audit 2024-09-13T00:24:45.075203+0000 mon.a (mon.0) 395 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cb032137-609e-41f6-bfc7-e70b77ea9456"}]': finished 2024-09-13T00:24:46.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:45 smithi088 bash[24895]: cluster 2024-09-13T00:24:45.075301+0000 mon.a (mon.0) 396 : cluster [DBG] osdmap e22: 4 total, 3 up, 4 in 2024-09-13T00:24:46.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:45 smithi088 bash[24895]: audit 2024-09-13T00:24:45.075530+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:24:47.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:46 smithi077 bash[22596]: cluster 2024-09-13T00:24:45.216259+0000 mgr.y (mgr.14150) 189 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:47.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:46 smithi077 bash[22596]: audit 2024-09-13T00:24:45.688862+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.77:0/3569296304' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:24:47.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:46 smithi077 bash[32040]: cluster 2024-09-13T00:24:45.216259+0000 mgr.y (mgr.14150) 189 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:47.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:46 smithi077 bash[32040]: audit 2024-09-13T00:24:45.688862+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.77:0/3569296304' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:24:47.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:46 smithi088 bash[24895]: cluster 2024-09-13T00:24:45.216259+0000 mgr.y (mgr.14150) 189 : cluster [DBG] pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:47.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:46 smithi088 bash[24895]: audit 2024-09-13T00:24:45.688862+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.77:0/3569296304' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:24:49.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:48 smithi077 bash[22596]: cluster 2024-09-13T00:24:47.216732+0000 mgr.y (mgr.14150) 190 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:49.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:48 smithi077 bash[32040]: cluster 2024-09-13T00:24:47.216732+0000 mgr.y (mgr.14150) 190 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:49.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:48 smithi088 bash[24895]: cluster 2024-09-13T00:24:47.216732+0000 mgr.y (mgr.14150) 190 : cluster [DBG] pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:51.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:50 smithi077 bash[22596]: cluster 2024-09-13T00:24:49.217206+0000 mgr.y (mgr.14150) 191 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:51.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:50 smithi077 bash[32040]: cluster 2024-09-13T00:24:49.217206+0000 mgr.y (mgr.14150) 191 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:51.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:50 smithi088 bash[24895]: cluster 2024-09-13T00:24:49.217206+0000 mgr.y (mgr.14150) 191 : cluster [DBG] pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:53.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:52 smithi077 bash[32040]: cluster 2024-09-13T00:24:51.217807+0000 mgr.y (mgr.14150) 192 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:53.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:52 smithi077 bash[22596]: cluster 2024-09-13T00:24:51.217807+0000 mgr.y (mgr.14150) 192 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:53.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:52 smithi088 bash[24895]: cluster 2024-09-13T00:24:51.217807+0000 mgr.y (mgr.14150) 192 : cluster [DBG] pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:54.979 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:54 smithi077 bash[22596]: cluster 2024-09-13T00:24:53.218427+0000 mgr.y (mgr.14150) 193 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:54.979 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:54 smithi077 bash[32040]: cluster 2024-09-13T00:24:53.218427+0000 mgr.y (mgr.14150) 193 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:55.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:54 smithi088 bash[24895]: cluster 2024-09-13T00:24:53.218427+0000 mgr.y (mgr.14150) 193 : cluster [DBG] pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:56.938 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:56 smithi077 bash[22596]: cluster 2024-09-13T00:24:55.219021+0000 mgr.y (mgr.14150) 194 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:56.939 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:56 smithi077 bash[32040]: cluster 2024-09-13T00:24:55.219021+0000 mgr.y (mgr.14150) 194 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:57.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:56 smithi088 bash[24895]: cluster 2024-09-13T00:24:55.219021+0000 mgr.y (mgr.14150) 194 : cluster [DBG] pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:59.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:58 smithi077 bash[22596]: cluster 2024-09-13T00:24:57.219571+0000 mgr.y (mgr.14150) 195 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:59.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:58 smithi077 bash[22596]: audit 2024-09-13T00:24:58.026881+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T00:24:59.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:58 smithi077 bash[22596]: audit 2024-09-13T00:24:58.028233+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:59.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:58 smithi077 bash[32040]: cluster 2024-09-13T00:24:57.219571+0000 mgr.y (mgr.14150) 195 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:59.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:58 smithi077 bash[32040]: audit 2024-09-13T00:24:58.026881+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T00:24:59.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:58 smithi077 bash[32040]: audit 2024-09-13T00:24:58.028233+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:58 smithi088 bash[24895]: cluster 2024-09-13T00:24:57.219571+0000 mgr.y (mgr.14150) 195 : cluster [DBG] pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:24:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:58 smithi088 bash[24895]: audit 2024-09-13T00:24:58.026881+0000 mon.a (mon.0) 398 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T00:24:59.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:58 smithi088 bash[24895]: audit 2024-09-13T00:24:58.028233+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:24:59.885 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:24:59 smithi077 bash[32040]: cephadm 2024-09-13T00:24:58.029459+0000 mgr.y (mgr.14150) 196 : cephadm [INF] Deploying daemon osd.3 on smithi077 2024-09-13T00:24:59.885 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:24:59 smithi077 bash[22596]: cephadm 2024-09-13T00:24:58.029459+0000 mgr.y (mgr.14150) 196 : cephadm [INF] Deploying daemon osd.3 on smithi077 2024-09-13T00:25:00.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:24:59 smithi088 bash[24895]: cephadm 2024-09-13T00:24:58.029459+0000 mgr.y (mgr.14150) 196 : cephadm [INF] Deploying daemon osd.3 on smithi077 2024-09-13T00:25:00.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:00 smithi077 bash[22596]: cluster 2024-09-13T00:24:59.220067+0000 mgr.y (mgr.14150) 197 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:00 smithi077 bash[32040]: cluster 2024-09-13T00:24:59.220067+0000 mgr.y (mgr.14150) 197 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:01.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:00 smithi088 bash[24895]: cluster 2024-09-13T00:24:59.220067+0000 mgr.y (mgr.14150) 197 : cluster [DBG] pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:02.901 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:25:02 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:25:02.901 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:25:02 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:25:02.902 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:25:02 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:25:02.902 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:02 smithi077 bash[22596]: cluster 2024-09-13T00:25:01.220589+0000 mgr.y (mgr.14150) 198 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:02.902 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:02 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:25:02.902 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:02 smithi077 bash[32040]: cluster 2024-09-13T00:25:01.220589+0000 mgr.y (mgr.14150) 198 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:02.902 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:02 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:25:02.903 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:25:02 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:25:03.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:02 smithi088 bash[24895]: cluster 2024-09-13T00:25:01.220589+0000 mgr.y (mgr.14150) 198 : cluster [DBG] pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:03.280 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:25:03 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:25:03.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:03 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:25:03.280 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:25:03 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:25:03.281 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:25:03 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:25:03.281 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:25:03 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:25:03.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:03 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:25:04.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:03 smithi077 bash[22596]: audit 2024-09-13T00:25:03.172599+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:25:04.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:03 smithi077 bash[22596]: audit 2024-09-13T00:25:03.177484+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:04.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:03 smithi077 bash[22596]: audit 2024-09-13T00:25:03.182672+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:04.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:03 smithi077 bash[32040]: audit 2024-09-13T00:25:03.172599+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:25:04.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:03 smithi077 bash[32040]: audit 2024-09-13T00:25:03.177484+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:04.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:03 smithi077 bash[32040]: audit 2024-09-13T00:25:03.182672+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:04.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:03 smithi088 bash[24895]: audit 2024-09-13T00:25:03.172599+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:25:04.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:03 smithi088 bash[24895]: audit 2024-09-13T00:25:03.177484+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:04.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:03 smithi088 bash[24895]: audit 2024-09-13T00:25:03.182672+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:04.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:04 smithi077 bash[22596]: cluster 2024-09-13T00:25:03.220924+0000 mgr.y (mgr.14150) 199 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:04.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:04 smithi077 bash[32040]: cluster 2024-09-13T00:25:03.220924+0000 mgr.y (mgr.14150) 199 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:05.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:04 smithi088 bash[24895]: cluster 2024-09-13T00:25:03.220924+0000 mgr.y (mgr.14150) 199 : cluster [DBG] pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:07.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:06 smithi077 bash[22596]: cluster 2024-09-13T00:25:05.221417+0000 mgr.y (mgr.14150) 200 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:07.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:06 smithi077 bash[32040]: cluster 2024-09-13T00:25:05.221417+0000 mgr.y (mgr.14150) 200 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:07.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:06 smithi088 bash[24895]: cluster 2024-09-13T00:25:05.221417+0000 mgr.y (mgr.14150) 200 : cluster [DBG] pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:09.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:08 smithi077 bash[32040]: cluster 2024-09-13T00:25:07.221929+0000 mgr.y (mgr.14150) 201 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:09.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:08 smithi077 bash[22596]: cluster 2024-09-13T00:25:07.221929+0000 mgr.y (mgr.14150) 201 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:09.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:08 smithi088 bash[24895]: cluster 2024-09-13T00:25:07.221929+0000 mgr.y (mgr.14150) 201 : cluster [DBG] pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:10 smithi077 bash[32040]: cluster 2024-09-13T00:25:09.222276+0000 mgr.y (mgr.14150) 202 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:10 smithi077 bash[32040]: audit 2024-09-13T00:25:09.466277+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:10 smithi077 bash[32040]: audit 2024-09-13T00:25:09.470884+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:10 smithi077 bash[32040]: audit 2024-09-13T00:25:09.471828+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:25:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:10 smithi077 bash[32040]: audit 2024-09-13T00:25:09.472466+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:25:10.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:10 smithi077 bash[32040]: audit 2024-09-13T00:25:09.477246+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:10.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:10 smithi077 bash[32040]: audit 2024-09-13T00:25:10.258839+0000 mon.c (mon.2) 10 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2636296765,v1:172.21.15.77:6827/2636296765]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T00:25:10.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:10 smithi077 bash[32040]: audit 2024-09-13T00:25:10.259203+0000 mon.a (mon.0) 408 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T00:25:10.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:10 smithi077 bash[22596]: cluster 2024-09-13T00:25:09.222276+0000 mgr.y (mgr.14150) 202 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:10.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:10 smithi077 bash[22596]: audit 2024-09-13T00:25:09.466277+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:10.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:10 smithi077 bash[22596]: audit 2024-09-13T00:25:09.470884+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:10.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:10 smithi077 bash[22596]: audit 2024-09-13T00:25:09.471828+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:25:10.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:10 smithi077 bash[22596]: audit 2024-09-13T00:25:09.472466+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:25:10.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:10 smithi077 bash[22596]: audit 2024-09-13T00:25:09.477246+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:10.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:10 smithi077 bash[22596]: audit 2024-09-13T00:25:10.258839+0000 mon.c (mon.2) 10 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2636296765,v1:172.21.15.77:6827/2636296765]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T00:25:10.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:10 smithi077 bash[22596]: audit 2024-09-13T00:25:10.259203+0000 mon.a (mon.0) 408 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T00:25:10.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:10 smithi088 bash[24895]: cluster 2024-09-13T00:25:09.222276+0000 mgr.y (mgr.14150) 202 : cluster [DBG] pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:10.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:10 smithi088 bash[24895]: audit 2024-09-13T00:25:09.466277+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:10.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:10 smithi088 bash[24895]: audit 2024-09-13T00:25:09.470884+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:10.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:10 smithi088 bash[24895]: audit 2024-09-13T00:25:09.471828+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:25:10.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:10 smithi088 bash[24895]: audit 2024-09-13T00:25:09.472466+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:25:10.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:10 smithi088 bash[24895]: audit 2024-09-13T00:25:09.477246+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:10.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:10 smithi088 bash[24895]: audit 2024-09-13T00:25:10.258839+0000 mon.c (mon.2) 10 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2636296765,v1:172.21.15.77:6827/2636296765]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T00:25:10.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:10 smithi088 bash[24895]: audit 2024-09-13T00:25:10.259203+0000 mon.a (mon.0) 408 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T00:25:11.754 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:11 smithi077 bash[22596]: audit 2024-09-13T00:25:10.718407+0000 mon.a (mon.0) 409 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T00:25:11.754 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:11 smithi077 bash[22596]: cluster 2024-09-13T00:25:10.718578+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-09-13T00:25:11.754 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:11 smithi077 bash[22596]: audit 2024-09-13T00:25:10.718895+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:25:11.755 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:11 smithi077 bash[22596]: audit 2024-09-13T00:25:10.720171+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2636296765,v1:172.21.15.77:6827/2636296765]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:25:11.755 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:11 smithi077 bash[22596]: audit 2024-09-13T00:25:10.720842+0000 mon.a (mon.0) 412 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:25:11.755 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:11 smithi077 bash[32040]: audit 2024-09-13T00:25:10.718407+0000 mon.a (mon.0) 409 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T00:25:11.755 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:11 smithi077 bash[32040]: cluster 2024-09-13T00:25:10.718578+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-09-13T00:25:11.755 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:11 smithi077 bash[32040]: audit 2024-09-13T00:25:10.718895+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:25:11.755 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:11 smithi077 bash[32040]: audit 2024-09-13T00:25:10.720171+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2636296765,v1:172.21.15.77:6827/2636296765]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:25:11.755 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:11 smithi077 bash[32040]: audit 2024-09-13T00:25:10.720842+0000 mon.a (mon.0) 412 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:25:12.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:11 smithi088 bash[24895]: audit 2024-09-13T00:25:10.718407+0000 mon.a (mon.0) 409 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T00:25:12.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:11 smithi088 bash[24895]: cluster 2024-09-13T00:25:10.718578+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e23: 4 total, 3 up, 4 in 2024-09-13T00:25:12.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:11 smithi088 bash[24895]: audit 2024-09-13T00:25:10.718895+0000 mon.a (mon.0) 411 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:25:12.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:11 smithi088 bash[24895]: audit 2024-09-13T00:25:10.720171+0000 mon.c (mon.2) 11 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2636296765,v1:172.21.15.77:6827/2636296765]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:25:12.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:11 smithi088 bash[24895]: audit 2024-09-13T00:25:10.720842+0000 mon.a (mon.0) 412 : audit [INF] from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:25:13.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:12 smithi077 bash[22596]: cluster 2024-09-13T00:25:11.222792+0000 mgr.y (mgr.14150) 203 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:13.083 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:12 smithi077 bash[22596]: audit 2024-09-13T00:25:11.721000+0000 mon.a (mon.0) 413 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]': finished 2024-09-13T00:25:13.083 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:12 smithi077 bash[22596]: cluster 2024-09-13T00:25:11.721122+0000 mon.a (mon.0) 414 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-09-13T00:25:13.083 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:12 smithi077 bash[22596]: audit 2024-09-13T00:25:11.721898+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:25:13.083 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:12 smithi077 bash[22596]: audit 2024-09-13T00:25:11.732044+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:25:13.083 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:12 smithi077 bash[32040]: cluster 2024-09-13T00:25:11.222792+0000 mgr.y (mgr.14150) 203 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:13.083 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:12 smithi077 bash[32040]: audit 2024-09-13T00:25:11.721000+0000 mon.a (mon.0) 413 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]': finished 2024-09-13T00:25:13.083 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:12 smithi077 bash[32040]: cluster 2024-09-13T00:25:11.721122+0000 mon.a (mon.0) 414 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-09-13T00:25:13.084 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:12 smithi077 bash[32040]: audit 2024-09-13T00:25:11.721898+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:25:13.084 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:12 smithi077 bash[32040]: audit 2024-09-13T00:25:11.732044+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:25:13.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:12 smithi088 bash[24895]: cluster 2024-09-13T00:25:11.222792+0000 mgr.y (mgr.14150) 203 : cluster [DBG] pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 873 MiB used, 267 GiB / 268 GiB avail 2024-09-13T00:25:13.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:12 smithi088 bash[24895]: audit 2024-09-13T00:25:11.721000+0000 mon.a (mon.0) 413 : audit [INF] from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]': finished 2024-09-13T00:25:13.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:12 smithi088 bash[24895]: cluster 2024-09-13T00:25:11.721122+0000 mon.a (mon.0) 414 : cluster [DBG] osdmap e24: 4 total, 3 up, 4 in 2024-09-13T00:25:13.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:12 smithi088 bash[24895]: audit 2024-09-13T00:25:11.721898+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:25:13.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:12 smithi088 bash[24895]: audit 2024-09-13T00:25:11.732044+0000 mon.a (mon.0) 416 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:25:13.874 INFO:teuthology.orchestra.run.smithi077.stdout:Created osd(s) 3 on host 'smithi077' 2024-09-13T00:25:14.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:13 smithi077 bash[32040]: cluster 2024-09-13T00:25:11.211133+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:25:14.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:13 smithi077 bash[32040]: cluster 2024-09-13T00:25:11.211223+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:25:14.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:13 smithi077 bash[32040]: cluster 2024-09-13T00:25:12.730472+0000 mon.a (mon.0) 417 : cluster [INF] osd.3 [v2:172.21.15.77:6826/2636296765,v1:172.21.15.77:6827/2636296765] boot 2024-09-13T00:25:14.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:13 smithi077 bash[32040]: cluster 2024-09-13T00:25:12.730625+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2024-09-13T00:25:14.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:13 smithi077 bash[32040]: audit 2024-09-13T00:25:12.731020+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:25:14.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:13 smithi077 bash[22596]: cluster 2024-09-13T00:25:11.211133+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:25:14.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:13 smithi077 bash[22596]: cluster 2024-09-13T00:25:11.211223+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:25:14.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:13 smithi077 bash[22596]: cluster 2024-09-13T00:25:12.730472+0000 mon.a (mon.0) 417 : cluster [INF] osd.3 [v2:172.21.15.77:6826/2636296765,v1:172.21.15.77:6827/2636296765] boot 2024-09-13T00:25:14.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:13 smithi077 bash[22596]: cluster 2024-09-13T00:25:12.730625+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2024-09-13T00:25:14.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:13 smithi077 bash[22596]: audit 2024-09-13T00:25:12.731020+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:25:14.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:13 smithi088 bash[24895]: cluster 2024-09-13T00:25:11.211133+0000 osd.3 (osd.3) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:25:14.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:13 smithi088 bash[24895]: cluster 2024-09-13T00:25:11.211223+0000 osd.3 (osd.3) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:25:14.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:13 smithi088 bash[24895]: cluster 2024-09-13T00:25:12.730472+0000 mon.a (mon.0) 417 : cluster [INF] osd.3 [v2:172.21.15.77:6826/2636296765,v1:172.21.15.77:6827/2636296765] boot 2024-09-13T00:25:14.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:13 smithi088 bash[24895]: cluster 2024-09-13T00:25:12.730625+0000 mon.a (mon.0) 418 : cluster [DBG] osdmap e25: 4 total, 4 up, 4 in 2024-09-13T00:25:14.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:13 smithi088 bash[24895]: audit 2024-09-13T00:25:12.731020+0000 mon.a (mon.0) 419 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:25:14.681 DEBUG:teuthology.orchestra.run.smithi077:osd.3> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.3.service 2024-09-13T00:25:14.684 INFO:tasks.cephadm:Deploying osd.4 on smithi088 with /dev/vg_nvme/lv_4... 2024-09-13T00:25:14.684 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-09-13T00:25:15.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:14 smithi077 bash[22596]: cluster 2024-09-13T00:25:13.223364+0000 mgr.y (mgr.14150) 204 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:15.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:14 smithi077 bash[22596]: cluster 2024-09-13T00:25:13.749827+0000 mon.a (mon.0) 420 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-09-13T00:25:15.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:14 smithi077 bash[22596]: audit 2024-09-13T00:25:13.854771+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:25:15.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:14 smithi077 bash[22596]: audit 2024-09-13T00:25:13.861161+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:15.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:14 smithi077 bash[22596]: audit 2024-09-13T00:25:13.871736+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:15.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:14 smithi077 bash[32040]: cluster 2024-09-13T00:25:13.223364+0000 mgr.y (mgr.14150) 204 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:15.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:14 smithi077 bash[32040]: cluster 2024-09-13T00:25:13.749827+0000 mon.a (mon.0) 420 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-09-13T00:25:15.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:14 smithi077 bash[32040]: audit 2024-09-13T00:25:13.854771+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:25:15.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:14 smithi077 bash[32040]: audit 2024-09-13T00:25:13.861161+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:15.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:14 smithi077 bash[32040]: audit 2024-09-13T00:25:13.871736+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:15.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:14 smithi088 bash[24895]: cluster 2024-09-13T00:25:13.223364+0000 mgr.y (mgr.14150) 204 : cluster [DBG] pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:15.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:14 smithi088 bash[24895]: cluster 2024-09-13T00:25:13.749827+0000 mon.a (mon.0) 420 : cluster [DBG] osdmap e26: 4 total, 4 up, 4 in 2024-09-13T00:25:15.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:14 smithi088 bash[24895]: audit 2024-09-13T00:25:13.854771+0000 mon.a (mon.0) 421 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:25:15.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:14 smithi088 bash[24895]: audit 2024-09-13T00:25:13.861161+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:15.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:14 smithi088 bash[24895]: audit 2024-09-13T00:25:13.871736+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:17.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:16 smithi077 bash[22596]: cluster 2024-09-13T00:25:15.223924+0000 mgr.y (mgr.14150) 205 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:17.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:16 smithi077 bash[32040]: cluster 2024-09-13T00:25:15.223924+0000 mgr.y (mgr.14150) 205 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:17.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:16 smithi088 bash[24895]: cluster 2024-09-13T00:25:15.223924+0000 mgr.y (mgr.14150) 205 : cluster [DBG] pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:19.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:18 smithi077 bash[22596]: cluster 2024-09-13T00:25:17.224570+0000 mgr.y (mgr.14150) 206 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:19.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:18 smithi077 bash[32040]: cluster 2024-09-13T00:25:17.224570+0000 mgr.y (mgr.14150) 206 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:19.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:18 smithi088 bash[24895]: cluster 2024-09-13T00:25:17.224570+0000 mgr.y (mgr.14150) 206 : cluster [DBG] pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:19.401 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T00:25:20.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:20 smithi088 bash[24895]: cluster 2024-09-13T00:25:19.225220+0000 mgr.y (mgr.14150) 207 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:21.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:20 smithi077 bash[22596]: cluster 2024-09-13T00:25:19.225220+0000 mgr.y (mgr.14150) 207 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:21.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:20 smithi077 bash[32040]: cluster 2024-09-13T00:25:19.225220+0000 mgr.y (mgr.14150) 207 : cluster [DBG] pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:21.993 INFO:teuthology.orchestra.run.smithi088.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-09-13T00:25:22.139 INFO:teuthology.orchestra.run.smithi088.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-09-13T00:25:22.158 INFO:teuthology.orchestra.run.smithi088.stderr: stderr: 10+0 records in 2024-09-13T00:25:22.159 INFO:teuthology.orchestra.run.smithi088.stderr:10+0 records out 2024-09-13T00:25:22.159 INFO:teuthology.orchestra.run.smithi088.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0158806 s, 660 MB/s 2024-09-13T00:25:22.159 INFO:teuthology.orchestra.run.smithi088.stderr:--> Zapping successful for: 2024-09-13T00:25:22.729 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi088:vg_nvme/lv_4 2024-09-13T00:25:23.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:22 smithi077 bash[22596]: cluster 2024-09-13T00:25:21.225779+0000 mgr.y (mgr.14150) 208 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:23.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:22 smithi077 bash[32040]: cluster 2024-09-13T00:25:21.225779+0000 mgr.y (mgr.14150) 208 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:23.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:22 smithi088 bash[24895]: cluster 2024-09-13T00:25:21.225779+0000 mgr.y (mgr.14150) 208 : cluster [DBG] pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:25.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[22596]: cluster 2024-09-13T00:25:23.226194+0000 mgr.y (mgr.14150) 209 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:25.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[22596]: audit 2024-09-13T00:25:24.231722+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:25.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[22596]: audit 2024-09-13T00:25:24.240921+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:25.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[22596]: audit 2024-09-13T00:25:24.243325+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:25:25.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[22596]: audit 2024-09-13T00:25:24.245226+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:25:25.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[22596]: audit 2024-09-13T00:25:24.246854+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:25:25.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[22596]: audit 2024-09-13T00:25:24.248473+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:25:25.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[22596]: audit 2024-09-13T00:25:24.258002+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:25.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[22596]: audit 2024-09-13T00:25:24.260965+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:25:25.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[22596]: audit 2024-09-13T00:25:24.262777+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:25:25.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[22596]: audit 2024-09-13T00:25:24.272171+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:25.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[32040]: cluster 2024-09-13T00:25:23.226194+0000 mgr.y (mgr.14150) 209 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:25.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[32040]: audit 2024-09-13T00:25:24.231722+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:25.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[32040]: audit 2024-09-13T00:25:24.240921+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:25.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[32040]: audit 2024-09-13T00:25:24.243325+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:25:25.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[32040]: audit 2024-09-13T00:25:24.245226+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:25:25.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[32040]: audit 2024-09-13T00:25:24.246854+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:25:25.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[32040]: audit 2024-09-13T00:25:24.248473+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:25:25.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[32040]: audit 2024-09-13T00:25:24.258002+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:25.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[32040]: audit 2024-09-13T00:25:24.260965+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:25:25.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[32040]: audit 2024-09-13T00:25:24.262777+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:25:25.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:24 smithi077 bash[32040]: audit 2024-09-13T00:25:24.272171+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:25.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:24 smithi088 bash[24895]: cluster 2024-09-13T00:25:23.226194+0000 mgr.y (mgr.14150) 209 : cluster [DBG] pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:25.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:24 smithi088 bash[24895]: audit 2024-09-13T00:25:24.231722+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:25.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:24 smithi088 bash[24895]: audit 2024-09-13T00:25:24.240921+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:25.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:24 smithi088 bash[24895]: audit 2024-09-13T00:25:24.243325+0000 mon.a (mon.0) 426 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:25:25.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:24 smithi088 bash[24895]: audit 2024-09-13T00:25:24.245226+0000 mon.a (mon.0) 427 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:25:25.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:24 smithi088 bash[24895]: audit 2024-09-13T00:25:24.246854+0000 mon.a (mon.0) 428 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:25:25.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:24 smithi088 bash[24895]: audit 2024-09-13T00:25:24.248473+0000 mon.a (mon.0) 429 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:25:25.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:24 smithi088 bash[24895]: audit 2024-09-13T00:25:24.258002+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:25.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:24 smithi088 bash[24895]: audit 2024-09-13T00:25:24.260965+0000 mon.a (mon.0) 431 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:25:25.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:24 smithi088 bash[24895]: audit 2024-09-13T00:25:24.262777+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:25:25.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:24 smithi088 bash[24895]: audit 2024-09-13T00:25:24.272171+0000 mon.a (mon.0) 433 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:26.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:25 smithi077 bash[22596]: cephadm 2024-09-13T00:25:24.222918+0000 mgr.y (mgr.14150) 210 : cephadm [INF] Detected new or changed devices on smithi077 2024-09-13T00:25:26.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:25 smithi077 bash[22596]: cephadm 2024-09-13T00:25:24.249554+0000 mgr.y (mgr.14150) 211 : cephadm [INF] Adjusting osd_memory_target on smithi077 to 4060M 2024-09-13T00:25:26.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:25 smithi077 bash[32040]: cephadm 2024-09-13T00:25:24.222918+0000 mgr.y (mgr.14150) 210 : cephadm [INF] Detected new or changed devices on smithi077 2024-09-13T00:25:26.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:25 smithi077 bash[32040]: cephadm 2024-09-13T00:25:24.249554+0000 mgr.y (mgr.14150) 211 : cephadm [INF] Adjusting osd_memory_target on smithi077 to 4060M 2024-09-13T00:25:26.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:25 smithi088 bash[24895]: cephadm 2024-09-13T00:25:24.222918+0000 mgr.y (mgr.14150) 210 : cephadm [INF] Detected new or changed devices on smithi077 2024-09-13T00:25:26.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:25 smithi088 bash[24895]: cephadm 2024-09-13T00:25:24.249554+0000 mgr.y (mgr.14150) 211 : cephadm [INF] Adjusting osd_memory_target on smithi077 to 4060M 2024-09-13T00:25:27.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:26 smithi077 bash[22596]: cluster 2024-09-13T00:25:25.226718+0000 mgr.y (mgr.14150) 212 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:27.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:26 smithi077 bash[32040]: cluster 2024-09-13T00:25:25.226718+0000 mgr.y (mgr.14150) 212 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:27.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:26 smithi088 bash[24895]: cluster 2024-09-13T00:25:25.226718+0000 mgr.y (mgr.14150) 212 : cluster [DBG] pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:27.441 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T00:25:29.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:28 smithi077 bash[22596]: cluster 2024-09-13T00:25:27.227237+0000 mgr.y (mgr.14150) 213 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:29.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:28 smithi077 bash[32040]: cluster 2024-09-13T00:25:27.227237+0000 mgr.y (mgr.14150) 213 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:29.097 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:28 smithi088 bash[24895]: cluster 2024-09-13T00:25:27.227237+0000 mgr.y (mgr.14150) 213 : cluster [DBG] pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:31.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:30 smithi088 bash[24895]: cluster 2024-09-13T00:25:29.227744+0000 mgr.y (mgr.14150) 214 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:31.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:30 smithi088 bash[24895]: audit 2024-09-13T00:25:30.132866+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:25:31.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:30 smithi088 bash[24895]: audit 2024-09-13T00:25:30.138818+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:25:31.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:30 smithi088 bash[24895]: audit 2024-09-13T00:25:30.140594+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:25:31.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:30 smithi077 bash[22596]: cluster 2024-09-13T00:25:29.227744+0000 mgr.y (mgr.14150) 214 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:31.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:30 smithi077 bash[22596]: audit 2024-09-13T00:25:30.132866+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:25:31.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:30 smithi077 bash[22596]: audit 2024-09-13T00:25:30.138818+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:25:31.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:30 smithi077 bash[22596]: audit 2024-09-13T00:25:30.140594+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:25:31.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:30 smithi077 bash[32040]: cluster 2024-09-13T00:25:29.227744+0000 mgr.y (mgr.14150) 214 : cluster [DBG] pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:31.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:30 smithi077 bash[32040]: audit 2024-09-13T00:25:30.132866+0000 mon.a (mon.0) 434 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:25:31.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:30 smithi077 bash[32040]: audit 2024-09-13T00:25:30.138818+0000 mon.a (mon.0) 435 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:25:31.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:30 smithi077 bash[32040]: audit 2024-09-13T00:25:30.140594+0000 mon.a (mon.0) 436 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:25:32.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:31 smithi088 bash[24895]: audit 2024-09-13T00:25:30.126154+0000 mgr.y (mgr.14150) 215 : audit [DBG] from='client.24166 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:25:32.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:31 smithi077 bash[22596]: audit 2024-09-13T00:25:30.126154+0000 mgr.y (mgr.14150) 215 : audit [DBG] from='client.24166 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:25:32.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:31 smithi077 bash[32040]: audit 2024-09-13T00:25:30.126154+0000 mgr.y (mgr.14150) 215 : audit [DBG] from='client.24166 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:25:33.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:32 smithi088 bash[24895]: cluster 2024-09-13T00:25:31.228269+0000 mgr.y (mgr.14150) 216 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:33.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:32 smithi077 bash[22596]: cluster 2024-09-13T00:25:31.228269+0000 mgr.y (mgr.14150) 216 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:33.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:32 smithi077 bash[32040]: cluster 2024-09-13T00:25:31.228269+0000 mgr.y (mgr.14150) 216 : cluster [DBG] pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:35.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:34 smithi088 bash[24895]: cluster 2024-09-13T00:25:33.228849+0000 mgr.y (mgr.14150) 217 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:35.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:34 smithi077 bash[22596]: cluster 2024-09-13T00:25:33.228849+0000 mgr.y (mgr.14150) 217 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:35.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:34 smithi077 bash[32040]: cluster 2024-09-13T00:25:33.228849+0000 mgr.y (mgr.14150) 217 : cluster [DBG] pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:37.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:36 smithi088 bash[24895]: cluster 2024-09-13T00:25:35.229470+0000 mgr.y (mgr.14150) 218 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:37.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:36 smithi077 bash[22596]: cluster 2024-09-13T00:25:35.229470+0000 mgr.y (mgr.14150) 218 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:37.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:36 smithi077 bash[32040]: cluster 2024-09-13T00:25:35.229470+0000 mgr.y (mgr.14150) 218 : cluster [DBG] pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:39.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:38 smithi088 bash[24895]: cluster 2024-09-13T00:25:37.230083+0000 mgr.y (mgr.14150) 219 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:39.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:38 smithi088 bash[24895]: audit 2024-09-13T00:25:38.084228+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.88:0/282096710' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cd38603e-5cdc-42b6-b8c8-656d44df0ed5"}]: dispatch 2024-09-13T00:25:39.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:38 smithi088 bash[24895]: audit 2024-09-13T00:25:38.085522+0000 mon.a (mon.0) 437 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cd38603e-5cdc-42b6-b8c8-656d44df0ed5"}]: dispatch 2024-09-13T00:25:39.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:38 smithi088 bash[24895]: audit 2024-09-13T00:25:38.094440+0000 mon.a (mon.0) 438 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cd38603e-5cdc-42b6-b8c8-656d44df0ed5"}]': finished 2024-09-13T00:25:39.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:38 smithi088 bash[24895]: cluster 2024-09-13T00:25:38.094595+0000 mon.a (mon.0) 439 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2024-09-13T00:25:39.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:38 smithi088 bash[24895]: audit 2024-09-13T00:25:38.094860+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:25:39.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:38 smithi088 bash[24895]: audit 2024-09-13T00:25:38.712725+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.88:0/4080508518' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:25:39.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:38 smithi077 bash[22596]: cluster 2024-09-13T00:25:37.230083+0000 mgr.y (mgr.14150) 219 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:39.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:38 smithi077 bash[22596]: audit 2024-09-13T00:25:38.084228+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.88:0/282096710' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cd38603e-5cdc-42b6-b8c8-656d44df0ed5"}]: dispatch 2024-09-13T00:25:39.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:38 smithi077 bash[22596]: audit 2024-09-13T00:25:38.085522+0000 mon.a (mon.0) 437 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cd38603e-5cdc-42b6-b8c8-656d44df0ed5"}]: dispatch 2024-09-13T00:25:39.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:38 smithi077 bash[22596]: audit 2024-09-13T00:25:38.094440+0000 mon.a (mon.0) 438 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cd38603e-5cdc-42b6-b8c8-656d44df0ed5"}]': finished 2024-09-13T00:25:39.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:38 smithi077 bash[22596]: cluster 2024-09-13T00:25:38.094595+0000 mon.a (mon.0) 439 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2024-09-13T00:25:39.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:38 smithi077 bash[22596]: audit 2024-09-13T00:25:38.094860+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:25:39.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:38 smithi077 bash[22596]: audit 2024-09-13T00:25:38.712725+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.88:0/4080508518' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:25:39.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:38 smithi077 bash[32040]: cluster 2024-09-13T00:25:37.230083+0000 mgr.y (mgr.14150) 219 : cluster [DBG] pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:39.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:38 smithi077 bash[32040]: audit 2024-09-13T00:25:38.084228+0000 mon.b (mon.1) 9 : audit [INF] from='client.? 172.21.15.88:0/282096710' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cd38603e-5cdc-42b6-b8c8-656d44df0ed5"}]: dispatch 2024-09-13T00:25:39.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:38 smithi077 bash[32040]: audit 2024-09-13T00:25:38.085522+0000 mon.a (mon.0) 437 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "cd38603e-5cdc-42b6-b8c8-656d44df0ed5"}]: dispatch 2024-09-13T00:25:39.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:38 smithi077 bash[32040]: audit 2024-09-13T00:25:38.094440+0000 mon.a (mon.0) 438 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "cd38603e-5cdc-42b6-b8c8-656d44df0ed5"}]': finished 2024-09-13T00:25:39.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:38 smithi077 bash[32040]: cluster 2024-09-13T00:25:38.094595+0000 mon.a (mon.0) 439 : cluster [DBG] osdmap e27: 5 total, 4 up, 5 in 2024-09-13T00:25:39.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:38 smithi077 bash[32040]: audit 2024-09-13T00:25:38.094860+0000 mon.a (mon.0) 440 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:25:39.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:38 smithi077 bash[32040]: audit 2024-09-13T00:25:38.712725+0000 mon.b (mon.1) 10 : audit [DBG] from='client.? 172.21.15.88:0/4080508518' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:25:41.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:40 smithi088 bash[24895]: cluster 2024-09-13T00:25:39.230720+0000 mgr.y (mgr.14150) 220 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:41.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:40 smithi077 bash[22596]: cluster 2024-09-13T00:25:39.230720+0000 mgr.y (mgr.14150) 220 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:41.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:40 smithi077 bash[32040]: cluster 2024-09-13T00:25:39.230720+0000 mgr.y (mgr.14150) 220 : cluster [DBG] pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:43.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:42 smithi088 bash[24895]: cluster 2024-09-13T00:25:41.231298+0000 mgr.y (mgr.14150) 221 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:43.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:42 smithi077 bash[22596]: cluster 2024-09-13T00:25:41.231298+0000 mgr.y (mgr.14150) 221 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:43.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:42 smithi077 bash[32040]: cluster 2024-09-13T00:25:41.231298+0000 mgr.y (mgr.14150) 221 : cluster [DBG] pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:45.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:44 smithi088 bash[24895]: cluster 2024-09-13T00:25:43.231874+0000 mgr.y (mgr.14150) 222 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:45.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:44 smithi077 bash[32040]: cluster 2024-09-13T00:25:43.231874+0000 mgr.y (mgr.14150) 222 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:45.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:44 smithi077 bash[22596]: cluster 2024-09-13T00:25:43.231874+0000 mgr.y (mgr.14150) 222 : cluster [DBG] pgmap v191: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:47.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:46 smithi088 bash[24895]: cluster 2024-09-13T00:25:45.232457+0000 mgr.y (mgr.14150) 223 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:47.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:46 smithi077 bash[22596]: cluster 2024-09-13T00:25:45.232457+0000 mgr.y (mgr.14150) 223 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:47.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:46 smithi077 bash[32040]: cluster 2024-09-13T00:25:45.232457+0000 mgr.y (mgr.14150) 223 : cluster [DBG] pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:49.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:48 smithi088 bash[24895]: cluster 2024-09-13T00:25:47.233088+0000 mgr.y (mgr.14150) 224 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:49.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:48 smithi077 bash[22596]: cluster 2024-09-13T00:25:47.233088+0000 mgr.y (mgr.14150) 224 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:49.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:48 smithi077 bash[32040]: cluster 2024-09-13T00:25:47.233088+0000 mgr.y (mgr.14150) 224 : cluster [DBG] pgmap v193: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:50.095 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:49 smithi088 bash[24895]: audit 2024-09-13T00:25:49.748205+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T00:25:50.096 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:49 smithi088 bash[24895]: audit 2024-09-13T00:25:49.749894+0000 mon.a (mon.0) 442 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:25:50.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:49 smithi077 bash[22596]: audit 2024-09-13T00:25:49.748205+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T00:25:50.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:49 smithi077 bash[22596]: audit 2024-09-13T00:25:49.749894+0000 mon.a (mon.0) 442 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:25:50.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:49 smithi077 bash[32040]: audit 2024-09-13T00:25:49.748205+0000 mon.a (mon.0) 441 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T00:25:50.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:49 smithi077 bash[32040]: audit 2024-09-13T00:25:49.749894+0000 mon.a (mon.0) 442 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:25:51.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:50 smithi088 bash[24895]: cluster 2024-09-13T00:25:49.233716+0000 mgr.y (mgr.14150) 225 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:51.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:50 smithi088 bash[24895]: cephadm 2024-09-13T00:25:49.751306+0000 mgr.y (mgr.14150) 226 : cephadm [INF] Deploying daemon osd.4 on smithi088 2024-09-13T00:25:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:50 smithi077 bash[22596]: cluster 2024-09-13T00:25:49.233716+0000 mgr.y (mgr.14150) 225 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:50 smithi077 bash[22596]: cephadm 2024-09-13T00:25:49.751306+0000 mgr.y (mgr.14150) 226 : cephadm [INF] Deploying daemon osd.4 on smithi088 2024-09-13T00:25:51.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:50 smithi077 bash[32040]: cluster 2024-09-13T00:25:49.233716+0000 mgr.y (mgr.14150) 225 : cluster [DBG] pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:51.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:50 smithi077 bash[32040]: cephadm 2024-09-13T00:25:49.751306+0000 mgr.y (mgr.14150) 226 : cephadm [INF] Deploying daemon osd.4 on smithi088 2024-09-13T00:25:53.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:52 smithi088 bash[24895]: cluster 2024-09-13T00:25:51.234351+0000 mgr.y (mgr.14150) 227 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:53.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:52 smithi077 bash[22596]: cluster 2024-09-13T00:25:51.234351+0000 mgr.y (mgr.14150) 227 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:53.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:52 smithi077 bash[32040]: cluster 2024-09-13T00:25:51.234351+0000 mgr.y (mgr.14150) 227 : cluster [DBG] pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:53.435 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:53 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:25:53.435 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:25:53 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:25:53.771 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:25:53 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:25:53.771 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:53 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:25:54.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:53 smithi088 bash[24895]: audit 2024-09-13T00:25:53.692214+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:25:54.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:53 smithi088 bash[24895]: audit 2024-09-13T00:25:53.699538+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:54.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:53 smithi088 bash[24895]: audit 2024-09-13T00:25:53.707480+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:53 smithi077 bash[22596]: audit 2024-09-13T00:25:53.692214+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:25:54.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:53 smithi077 bash[22596]: audit 2024-09-13T00:25:53.699538+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:54.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:53 smithi077 bash[22596]: audit 2024-09-13T00:25:53.707480+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:53 smithi077 bash[32040]: audit 2024-09-13T00:25:53.692214+0000 mon.a (mon.0) 443 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:25:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:53 smithi077 bash[32040]: audit 2024-09-13T00:25:53.699538+0000 mon.a (mon.0) 444 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:53 smithi077 bash[32040]: audit 2024-09-13T00:25:53.707480+0000 mon.a (mon.0) 445 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:55.030 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:54 smithi088 bash[24895]: cluster 2024-09-13T00:25:53.234947+0000 mgr.y (mgr.14150) 228 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:55.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:54 smithi077 bash[22596]: cluster 2024-09-13T00:25:53.234947+0000 mgr.y (mgr.14150) 228 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:55.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:54 smithi077 bash[32040]: cluster 2024-09-13T00:25:53.234947+0000 mgr.y (mgr.14150) 228 : cluster [DBG] pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:57.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:56 smithi077 bash[22596]: cluster 2024-09-13T00:25:55.235494+0000 mgr.y (mgr.14150) 229 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:57.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:56 smithi077 bash[32040]: cluster 2024-09-13T00:25:55.235494+0000 mgr.y (mgr.14150) 229 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:57.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:56 smithi088 bash[24895]: cluster 2024-09-13T00:25:55.235494+0000 mgr.y (mgr.14150) 229 : cluster [DBG] pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:59.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:58 smithi077 bash[22596]: cluster 2024-09-13T00:25:57.236025+0000 mgr.y (mgr.14150) 230 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:59.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:58 smithi077 bash[22596]: audit 2024-09-13T00:25:58.732241+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:59.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:58 smithi077 bash[22596]: audit 2024-09-13T00:25:58.742805+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:59.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:58 smithi077 bash[32040]: cluster 2024-09-13T00:25:57.236025+0000 mgr.y (mgr.14150) 230 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:59.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:58 smithi077 bash[32040]: audit 2024-09-13T00:25:58.732241+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:59.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:58 smithi077 bash[32040]: audit 2024-09-13T00:25:58.742805+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:59.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:58 smithi088 bash[24895]: cluster 2024-09-13T00:25:57.236025+0000 mgr.y (mgr.14150) 230 : cluster [DBG] pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:25:59.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:58 smithi088 bash[24895]: audit 2024-09-13T00:25:58.732241+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:25:59.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:58 smithi088 bash[24895]: audit 2024-09-13T00:25:58.742805+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:00.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:59 smithi088 bash[24895]: audit 2024-09-13T00:25:59.156685+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:00.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:59 smithi088 bash[24895]: audit 2024-09-13T00:25:59.158724+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:26:00.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:25:59 smithi088 bash[24895]: audit 2024-09-13T00:25:59.169075+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:59 smithi077 bash[22596]: audit 2024-09-13T00:25:59.156685+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:00.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:59 smithi077 bash[22596]: audit 2024-09-13T00:25:59.158724+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:26:00.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:25:59 smithi077 bash[22596]: audit 2024-09-13T00:25:59.169075+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:59 smithi077 bash[32040]: audit 2024-09-13T00:25:59.156685+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:59 smithi077 bash[32040]: audit 2024-09-13T00:25:59.158724+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:26:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:25:59 smithi077 bash[32040]: audit 2024-09-13T00:25:59.169075+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:00 smithi077 bash[22596]: cluster 2024-09-13T00:25:59.236527+0000 mgr.y (mgr.14150) 231 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:26:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:00 smithi077 bash[22596]: audit 2024-09-13T00:25:59.911835+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.88:6800/3645723145,v1:172.21.15.88:6801/3645723145]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T00:26:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:00 smithi077 bash[22596]: audit 2024-09-13T00:25:59.913080+0000 mon.a (mon.0) 451 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T00:26:01.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:00 smithi077 bash[32040]: cluster 2024-09-13T00:25:59.236527+0000 mgr.y (mgr.14150) 231 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:26:01.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:00 smithi077 bash[32040]: audit 2024-09-13T00:25:59.911835+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.88:6800/3645723145,v1:172.21.15.88:6801/3645723145]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T00:26:01.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:00 smithi077 bash[32040]: audit 2024-09-13T00:25:59.913080+0000 mon.a (mon.0) 451 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T00:26:01.340 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:00 smithi088 bash[24895]: cluster 2024-09-13T00:25:59.236527+0000 mgr.y (mgr.14150) 231 : cluster [DBG] pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:26:01.340 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:00 smithi088 bash[24895]: audit 2024-09-13T00:25:59.911835+0000 mon.b (mon.1) 11 : audit [INF] from='osd.4 [v2:172.21.15.88:6800/3645723145,v1:172.21.15.88:6801/3645723145]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T00:26:01.340 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:00 smithi088 bash[24895]: audit 2024-09-13T00:25:59.913080+0000 mon.a (mon.0) 451 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T00:26:01.943 INFO:teuthology.orchestra.run.smithi088.stdout:Created osd(s) 4 on host 'smithi088' 2024-09-13T00:26:02.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:01 smithi077 bash[22596]: audit 2024-09-13T00:26:00.876440+0000 mon.a (mon.0) 452 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T00:26:02.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:01 smithi077 bash[22596]: cluster 2024-09-13T00:26:00.876555+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-09-13T00:26:02.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:01 smithi077 bash[22596]: audit 2024-09-13T00:26:00.877152+0000 mon.a (mon.0) 454 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:26:02.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:01 smithi077 bash[22596]: audit 2024-09-13T00:26:00.877268+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.88:6800/3645723145,v1:172.21.15.88:6801/3645723145]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:26:02.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:01 smithi077 bash[22596]: audit 2024-09-13T00:26:00.884494+0000 mon.a (mon.0) 455 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:26:02.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:01 smithi077 bash[32040]: audit 2024-09-13T00:26:00.876440+0000 mon.a (mon.0) 452 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T00:26:02.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:01 smithi077 bash[32040]: cluster 2024-09-13T00:26:00.876555+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-09-13T00:26:02.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:01 smithi077 bash[32040]: audit 2024-09-13T00:26:00.877152+0000 mon.a (mon.0) 454 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:26:02.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:01 smithi077 bash[32040]: audit 2024-09-13T00:26:00.877268+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.88:6800/3645723145,v1:172.21.15.88:6801/3645723145]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:26:02.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:01 smithi077 bash[32040]: audit 2024-09-13T00:26:00.884494+0000 mon.a (mon.0) 455 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:26:02.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:01 smithi088 bash[24895]: audit 2024-09-13T00:26:00.876440+0000 mon.a (mon.0) 452 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T00:26:02.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:01 smithi088 bash[24895]: cluster 2024-09-13T00:26:00.876555+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e28: 5 total, 4 up, 5 in 2024-09-13T00:26:02.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:01 smithi088 bash[24895]: audit 2024-09-13T00:26:00.877152+0000 mon.a (mon.0) 454 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:26:02.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:01 smithi088 bash[24895]: audit 2024-09-13T00:26:00.877268+0000 mon.b (mon.1) 12 : audit [INF] from='osd.4 [v2:172.21.15.88:6800/3645723145,v1:172.21.15.88:6801/3645723145]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:26:02.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:01 smithi088 bash[24895]: audit 2024-09-13T00:26:00.884494+0000 mon.a (mon.0) 455 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:26:02.441 DEBUG:teuthology.orchestra.run.smithi088:osd.4> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.4.service 2024-09-13T00:26:02.443 INFO:tasks.cephadm:Deploying osd.5 on smithi088 with /dev/vg_nvme/lv_3... 2024-09-13T00:26:02.444 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-09-13T00:26:03.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:02 smithi077 bash[22596]: cluster 2024-09-13T00:26:01.237034+0000 mgr.y (mgr.14150) 232 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:26:03.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:02 smithi077 bash[22596]: audit 2024-09-13T00:26:01.886798+0000 mon.a (mon.0) 456 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-13T00:26:03.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:02 smithi077 bash[22596]: cluster 2024-09-13T00:26:01.887005+0000 mon.a (mon.0) 457 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-09-13T00:26:03.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:02 smithi077 bash[22596]: audit 2024-09-13T00:26:01.887825+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:26:03.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:02 smithi077 bash[22596]: audit 2024-09-13T00:26:01.897872+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:26:03.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:02 smithi077 bash[22596]: audit 2024-09-13T00:26:01.922172+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:26:03.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:02 smithi077 bash[22596]: audit 2024-09-13T00:26:01.931247+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:03.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:02 smithi077 bash[22596]: audit 2024-09-13T00:26:01.941392+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:03.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:02 smithi077 bash[32040]: cluster 2024-09-13T00:26:01.237034+0000 mgr.y (mgr.14150) 232 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:26:03.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:02 smithi077 bash[32040]: audit 2024-09-13T00:26:01.886798+0000 mon.a (mon.0) 456 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-13T00:26:03.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:02 smithi077 bash[32040]: cluster 2024-09-13T00:26:01.887005+0000 mon.a (mon.0) 457 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-09-13T00:26:03.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:02 smithi077 bash[32040]: audit 2024-09-13T00:26:01.887825+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:26:03.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:02 smithi077 bash[32040]: audit 2024-09-13T00:26:01.897872+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:26:03.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:02 smithi077 bash[32040]: audit 2024-09-13T00:26:01.922172+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:26:03.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:02 smithi077 bash[32040]: audit 2024-09-13T00:26:01.931247+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:03.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:02 smithi077 bash[32040]: audit 2024-09-13T00:26:01.941392+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:03.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:02 smithi088 bash[24895]: cluster 2024-09-13T00:26:01.237034+0000 mgr.y (mgr.14150) 232 : cluster [DBG] pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-09-13T00:26:03.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:02 smithi088 bash[24895]: audit 2024-09-13T00:26:01.886798+0000 mon.a (mon.0) 456 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-13T00:26:03.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:02 smithi088 bash[24895]: cluster 2024-09-13T00:26:01.887005+0000 mon.a (mon.0) 457 : cluster [DBG] osdmap e29: 5 total, 4 up, 5 in 2024-09-13T00:26:03.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:02 smithi088 bash[24895]: audit 2024-09-13T00:26:01.887825+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:26:03.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:02 smithi088 bash[24895]: audit 2024-09-13T00:26:01.897872+0000 mon.a (mon.0) 459 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:26:03.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:02 smithi088 bash[24895]: audit 2024-09-13T00:26:01.922172+0000 mon.a (mon.0) 460 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:26:03.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:02 smithi088 bash[24895]: audit 2024-09-13T00:26:01.931247+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:03.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:02 smithi088 bash[24895]: audit 2024-09-13T00:26:01.941392+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:04.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:03 smithi077 bash[22596]: cluster 2024-09-13T00:26:00.948944+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:26:04.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:03 smithi077 bash[22596]: cluster 2024-09-13T00:26:00.949028+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:26:04.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:03 smithi077 bash[22596]: cluster 2024-09-13T00:26:02.891448+0000 mon.a (mon.0) 463 : cluster [INF] osd.4 [v2:172.21.15.88:6800/3645723145,v1:172.21.15.88:6801/3645723145] boot 2024-09-13T00:26:04.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:03 smithi077 bash[22596]: cluster 2024-09-13T00:26:02.891556+0000 mon.a (mon.0) 464 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2024-09-13T00:26:04.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:03 smithi077 bash[22596]: audit 2024-09-13T00:26:02.891863+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:26:04.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:03 smithi077 bash[32040]: cluster 2024-09-13T00:26:00.948944+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:26:04.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:03 smithi077 bash[32040]: cluster 2024-09-13T00:26:00.949028+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:26:04.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:03 smithi077 bash[32040]: cluster 2024-09-13T00:26:02.891448+0000 mon.a (mon.0) 463 : cluster [INF] osd.4 [v2:172.21.15.88:6800/3645723145,v1:172.21.15.88:6801/3645723145] boot 2024-09-13T00:26:04.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:03 smithi077 bash[32040]: cluster 2024-09-13T00:26:02.891556+0000 mon.a (mon.0) 464 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2024-09-13T00:26:04.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:03 smithi077 bash[32040]: audit 2024-09-13T00:26:02.891863+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:26:04.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:03 smithi088 bash[24895]: cluster 2024-09-13T00:26:00.948944+0000 osd.4 (osd.4) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:26:04.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:03 smithi088 bash[24895]: cluster 2024-09-13T00:26:00.949028+0000 osd.4 (osd.4) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:26:04.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:03 smithi088 bash[24895]: cluster 2024-09-13T00:26:02.891448+0000 mon.a (mon.0) 463 : cluster [INF] osd.4 [v2:172.21.15.88:6800/3645723145,v1:172.21.15.88:6801/3645723145] boot 2024-09-13T00:26:04.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:03 smithi088 bash[24895]: cluster 2024-09-13T00:26:02.891556+0000 mon.a (mon.0) 464 : cluster [DBG] osdmap e30: 5 total, 5 up, 5 in 2024-09-13T00:26:04.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:03 smithi088 bash[24895]: audit 2024-09-13T00:26:02.891863+0000 mon.a (mon.0) 465 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:26:05.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:04 smithi077 bash[22596]: cluster 2024-09-13T00:26:03.237610+0000 mgr.y (mgr.14150) 233 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:05.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:04 smithi077 bash[22596]: cluster 2024-09-13T00:26:03.896687+0000 mon.a (mon.0) 466 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-09-13T00:26:05.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:04 smithi077 bash[22596]: cluster 2024-09-13T00:26:04.898557+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-09-13T00:26:05.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:04 smithi077 bash[32040]: cluster 2024-09-13T00:26:03.237610+0000 mgr.y (mgr.14150) 233 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:05.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:04 smithi077 bash[32040]: cluster 2024-09-13T00:26:03.896687+0000 mon.a (mon.0) 466 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-09-13T00:26:05.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:04 smithi077 bash[32040]: cluster 2024-09-13T00:26:04.898557+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-09-13T00:26:05.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:04 smithi088 bash[24895]: cluster 2024-09-13T00:26:03.237610+0000 mgr.y (mgr.14150) 233 : cluster [DBG] pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:05.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:04 smithi088 bash[24895]: cluster 2024-09-13T00:26:03.896687+0000 mon.a (mon.0) 466 : cluster [DBG] osdmap e31: 5 total, 5 up, 5 in 2024-09-13T00:26:05.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:04 smithi088 bash[24895]: cluster 2024-09-13T00:26:04.898557+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e32: 5 total, 5 up, 5 in 2024-09-13T00:26:06.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:05 smithi088 bash[24895]: cluster 2024-09-13T00:26:05.238144+0000 mgr.y (mgr.14150) 234 : cluster [DBG] pgmap v207: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:06.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:05 smithi077 bash[22596]: cluster 2024-09-13T00:26:05.238144+0000 mgr.y (mgr.14150) 234 : cluster [DBG] pgmap v207: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:06.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:05 smithi077 bash[32040]: cluster 2024-09-13T00:26:05.238144+0000 mgr.y (mgr.14150) 234 : cluster [DBG] pgmap v207: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:06.694 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T00:26:08.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:08 smithi088 bash[24895]: cluster 2024-09-13T00:26:07.238836+0000 mgr.y (mgr.14150) 235 : cluster [DBG] pgmap v208: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:08.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:08 smithi077 bash[22596]: cluster 2024-09-13T00:26:07.238836+0000 mgr.y (mgr.14150) 235 : cluster [DBG] pgmap v208: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:08.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:08 smithi077 bash[32040]: cluster 2024-09-13T00:26:07.238836+0000 mgr.y (mgr.14150) 235 : cluster [DBG] pgmap v208: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:10.312 INFO:teuthology.orchestra.run.smithi088.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-09-13T00:26:10.518 INFO:teuthology.orchestra.run.smithi088.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-09-13T00:26:10.537 INFO:teuthology.orchestra.run.smithi088.stderr: stderr: 10+0 records in 2024-09-13T00:26:10.537 INFO:teuthology.orchestra.run.smithi088.stderr:10+0 records out 2024-09-13T00:26:10.537 INFO:teuthology.orchestra.run.smithi088.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0166413 s, 630 MB/s 2024-09-13T00:26:10.537 INFO:teuthology.orchestra.run.smithi088.stderr: stderr: 2024-09-13T00:26:10.538 INFO:teuthology.orchestra.run.smithi088.stderr:--> Zapping successful for: 2024-09-13T00:26:10.562 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:10 smithi088 bash[24895]: cluster 2024-09-13T00:26:09.239391+0000 mgr.y (mgr.14150) 236 : cluster [DBG] pgmap v209: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:10.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:10 smithi077 bash[32040]: cluster 2024-09-13T00:26:09.239391+0000 mgr.y (mgr.14150) 236 : cluster [DBG] pgmap v209: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:10.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:10 smithi077 bash[22596]: cluster 2024-09-13T00:26:09.239391+0000 mgr.y (mgr.14150) 236 : cluster [DBG] pgmap v209: 1 pgs: 1 peering; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:11.054 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi088:vg_nvme/lv_3 2024-09-13T00:26:11.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:11 smithi088 bash[24895]: cephadm 2024-09-13T00:26:10.560813+0000 mgr.y (mgr.14150) 237 : cephadm [INF] Detected new or changed devices on smithi088 2024-09-13T00:26:11.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:11 smithi088 bash[24895]: audit 2024-09-13T00:26:10.570874+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:11.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:11 smithi088 bash[24895]: audit 2024-09-13T00:26:10.580918+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:11.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:11 smithi088 bash[24895]: audit 2024-09-13T00:26:10.582933+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:26:11.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:11 smithi088 bash[24895]: cephadm 2024-09-13T00:26:10.584415+0000 mgr.y (mgr.14150) 238 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 17265M 2024-09-13T00:26:11.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:11 smithi088 bash[24895]: audit 2024-09-13T00:26:10.594304+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:11.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:11 smithi088 bash[24895]: audit 2024-09-13T00:26:10.596708+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:11.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:11 smithi088 bash[24895]: audit 2024-09-13T00:26:10.598372+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:26:11.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:11 smithi088 bash[24895]: audit 2024-09-13T00:26:10.609553+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[32040]: cephadm 2024-09-13T00:26:10.560813+0000 mgr.y (mgr.14150) 237 : cephadm [INF] Detected new or changed devices on smithi088 2024-09-13T00:26:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[32040]: audit 2024-09-13T00:26:10.570874+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[32040]: audit 2024-09-13T00:26:10.580918+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[32040]: audit 2024-09-13T00:26:10.582933+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:26:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[32040]: cephadm 2024-09-13T00:26:10.584415+0000 mgr.y (mgr.14150) 238 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 17265M 2024-09-13T00:26:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[32040]: audit 2024-09-13T00:26:10.594304+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[32040]: audit 2024-09-13T00:26:10.596708+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[32040]: audit 2024-09-13T00:26:10.598372+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:26:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[32040]: audit 2024-09-13T00:26:10.609553+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:12.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[22596]: cephadm 2024-09-13T00:26:10.560813+0000 mgr.y (mgr.14150) 237 : cephadm [INF] Detected new or changed devices on smithi088 2024-09-13T00:26:12.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[22596]: audit 2024-09-13T00:26:10.570874+0000 mon.a (mon.0) 468 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:12.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[22596]: audit 2024-09-13T00:26:10.580918+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:12.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[22596]: audit 2024-09-13T00:26:10.582933+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:26:12.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[22596]: cephadm 2024-09-13T00:26:10.584415+0000 mgr.y (mgr.14150) 238 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 17265M 2024-09-13T00:26:12.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[22596]: audit 2024-09-13T00:26:10.594304+0000 mon.a (mon.0) 471 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:12.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[22596]: audit 2024-09-13T00:26:10.596708+0000 mon.a (mon.0) 472 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:12.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[22596]: audit 2024-09-13T00:26:10.598372+0000 mon.a (mon.0) 473 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:26:12.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:11 smithi077 bash[22596]: audit 2024-09-13T00:26:10.609553+0000 mon.a (mon.0) 474 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:13.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:12 smithi077 bash[22596]: cluster 2024-09-13T00:26:11.239889+0000 mgr.y (mgr.14150) 239 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-13T00:26:13.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:12 smithi077 bash[32040]: cluster 2024-09-13T00:26:11.239889+0000 mgr.y (mgr.14150) 239 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-13T00:26:13.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:12 smithi088 bash[24895]: cluster 2024-09-13T00:26:11.239889+0000 mgr.y (mgr.14150) 239 : cluster [DBG] pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-13T00:26:15.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:14 smithi077 bash[22596]: cluster 2024-09-13T00:26:13.240471+0000 mgr.y (mgr.14150) 240 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-09-13T00:26:15.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:14 smithi077 bash[32040]: cluster 2024-09-13T00:26:13.240471+0000 mgr.y (mgr.14150) 240 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-09-13T00:26:15.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:14 smithi088 bash[24895]: cluster 2024-09-13T00:26:13.240471+0000 mgr.y (mgr.14150) 240 : cluster [DBG] pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2024-09-13T00:26:15.778 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T00:26:16.876 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:16 smithi088 bash[24895]: cluster 2024-09-13T00:26:15.241008+0000 mgr.y (mgr.14150) 241 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-09-13T00:26:17.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:16 smithi077 bash[22596]: cluster 2024-09-13T00:26:15.241008+0000 mgr.y (mgr.14150) 241 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-09-13T00:26:17.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:16 smithi077 bash[32040]: cluster 2024-09-13T00:26:15.241008+0000 mgr.y (mgr.14150) 241 : cluster [DBG] pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 56 KiB/s, 0 objects/s recovering 2024-09-13T00:26:19.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:18 smithi077 bash[32040]: cluster 2024-09-13T00:26:17.241607+0000 mgr.y (mgr.14150) 242 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T00:26:19.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:18 smithi077 bash[32040]: audit 2024-09-13T00:26:18.568607+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:26:19.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:18 smithi077 bash[32040]: audit 2024-09-13T00:26:18.574696+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:26:19.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:18 smithi077 bash[32040]: audit 2024-09-13T00:26:18.576320+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:19.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:18 smithi077 bash[22596]: cluster 2024-09-13T00:26:17.241607+0000 mgr.y (mgr.14150) 242 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T00:26:19.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:18 smithi077 bash[22596]: audit 2024-09-13T00:26:18.568607+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:26:19.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:18 smithi077 bash[22596]: audit 2024-09-13T00:26:18.574696+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:26:19.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:18 smithi077 bash[22596]: audit 2024-09-13T00:26:18.576320+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:19.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:18 smithi088 bash[24895]: cluster 2024-09-13T00:26:17.241607+0000 mgr.y (mgr.14150) 242 : cluster [DBG] pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T00:26:19.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:18 smithi088 bash[24895]: audit 2024-09-13T00:26:18.568607+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:26:19.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:18 smithi088 bash[24895]: audit 2024-09-13T00:26:18.574696+0000 mon.a (mon.0) 476 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:26:19.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:18 smithi088 bash[24895]: audit 2024-09-13T00:26:18.576320+0000 mon.a (mon.0) 477 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:20.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:19 smithi077 bash[32040]: audit 2024-09-13T00:26:18.562178+0000 mgr.y (mgr.14150) 243 : audit [DBG] from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:26:20.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:19 smithi077 bash[22596]: audit 2024-09-13T00:26:18.562178+0000 mgr.y (mgr.14150) 243 : audit [DBG] from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:26:20.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:19 smithi088 bash[24895]: audit 2024-09-13T00:26:18.562178+0000 mgr.y (mgr.14150) 243 : audit [DBG] from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:26:21.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:20 smithi077 bash[32040]: cluster 2024-09-13T00:26:19.242159+0000 mgr.y (mgr.14150) 244 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T00:26:21.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:20 smithi077 bash[22596]: cluster 2024-09-13T00:26:19.242159+0000 mgr.y (mgr.14150) 244 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T00:26:21.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:20 smithi088 bash[24895]: cluster 2024-09-13T00:26:19.242159+0000 mgr.y (mgr.14150) 244 : cluster [DBG] pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T00:26:23.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:22 smithi077 bash[22596]: cluster 2024-09-13T00:26:21.242735+0000 mgr.y (mgr.14150) 245 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T00:26:23.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:22 smithi077 bash[32040]: cluster 2024-09-13T00:26:21.242735+0000 mgr.y (mgr.14150) 245 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T00:26:23.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:22 smithi088 bash[24895]: cluster 2024-09-13T00:26:21.242735+0000 mgr.y (mgr.14150) 245 : cluster [DBG] pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T00:26:24.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:24 smithi088 bash[24895]: cluster 2024-09-13T00:26:23.243258+0000 mgr.y (mgr.14150) 246 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:25.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:24 smithi077 bash[22596]: cluster 2024-09-13T00:26:23.243258+0000 mgr.y (mgr.14150) 246 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:25.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:24 smithi077 bash[32040]: cluster 2024-09-13T00:26:23.243258+0000 mgr.y (mgr.14150) 246 : cluster [DBG] pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:27.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:26 smithi077 bash[22596]: cluster 2024-09-13T00:26:25.243778+0000 mgr.y (mgr.14150) 247 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:27.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:26 smithi077 bash[22596]: audit 2024-09-13T00:26:26.471312+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.88:0/1314223037' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4b81ff3-f257-4f63-a7a7-cbd1ff754544"}]: dispatch 2024-09-13T00:26:27.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:26 smithi077 bash[22596]: audit 2024-09-13T00:26:26.472655+0000 mon.a (mon.0) 478 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4b81ff3-f257-4f63-a7a7-cbd1ff754544"}]: dispatch 2024-09-13T00:26:27.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:26 smithi077 bash[22596]: audit 2024-09-13T00:26:26.481839+0000 mon.a (mon.0) 479 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d4b81ff3-f257-4f63-a7a7-cbd1ff754544"}]': finished 2024-09-13T00:26:27.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:26 smithi077 bash[22596]: cluster 2024-09-13T00:26:26.482087+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2024-09-13T00:26:27.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:26 smithi077 bash[22596]: audit 2024-09-13T00:26:26.482359+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:26:27.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:26 smithi077 bash[32040]: cluster 2024-09-13T00:26:25.243778+0000 mgr.y (mgr.14150) 247 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:27.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:26 smithi077 bash[32040]: audit 2024-09-13T00:26:26.471312+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.88:0/1314223037' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4b81ff3-f257-4f63-a7a7-cbd1ff754544"}]: dispatch 2024-09-13T00:26:27.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:26 smithi077 bash[32040]: audit 2024-09-13T00:26:26.472655+0000 mon.a (mon.0) 478 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4b81ff3-f257-4f63-a7a7-cbd1ff754544"}]: dispatch 2024-09-13T00:26:27.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:26 smithi077 bash[32040]: audit 2024-09-13T00:26:26.481839+0000 mon.a (mon.0) 479 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d4b81ff3-f257-4f63-a7a7-cbd1ff754544"}]': finished 2024-09-13T00:26:27.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:26 smithi077 bash[32040]: cluster 2024-09-13T00:26:26.482087+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2024-09-13T00:26:27.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:26 smithi077 bash[32040]: audit 2024-09-13T00:26:26.482359+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:26:27.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:26 smithi088 bash[24895]: cluster 2024-09-13T00:26:25.243778+0000 mgr.y (mgr.14150) 247 : cluster [DBG] pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:27.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:26 smithi088 bash[24895]: audit 2024-09-13T00:26:26.471312+0000 mon.b (mon.1) 13 : audit [INF] from='client.? 172.21.15.88:0/1314223037' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4b81ff3-f257-4f63-a7a7-cbd1ff754544"}]: dispatch 2024-09-13T00:26:27.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:26 smithi088 bash[24895]: audit 2024-09-13T00:26:26.472655+0000 mon.a (mon.0) 478 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d4b81ff3-f257-4f63-a7a7-cbd1ff754544"}]: dispatch 2024-09-13T00:26:27.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:26 smithi088 bash[24895]: audit 2024-09-13T00:26:26.481839+0000 mon.a (mon.0) 479 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d4b81ff3-f257-4f63-a7a7-cbd1ff754544"}]': finished 2024-09-13T00:26:27.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:26 smithi088 bash[24895]: cluster 2024-09-13T00:26:26.482087+0000 mon.a (mon.0) 480 : cluster [DBG] osdmap e33: 6 total, 5 up, 6 in 2024-09-13T00:26:27.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:26 smithi088 bash[24895]: audit 2024-09-13T00:26:26.482359+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:26:28.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:27 smithi077 bash[32040]: audit 2024-09-13T00:26:27.121132+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.88:0/3759000332' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:26:28.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:27 smithi077 bash[22596]: audit 2024-09-13T00:26:27.121132+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.88:0/3759000332' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:26:28.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:27 smithi088 bash[24895]: audit 2024-09-13T00:26:27.121132+0000 mon.b (mon.1) 14 : audit [DBG] from='client.? 172.21.15.88:0/3759000332' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:26:29.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:28 smithi077 bash[22596]: cluster 2024-09-13T00:26:27.244275+0000 mgr.y (mgr.14150) 248 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:29.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:28 smithi077 bash[32040]: cluster 2024-09-13T00:26:27.244275+0000 mgr.y (mgr.14150) 248 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:29.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:28 smithi088 bash[24895]: cluster 2024-09-13T00:26:27.244275+0000 mgr.y (mgr.14150) 248 : cluster [DBG] pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:31.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:30 smithi077 bash[22596]: cluster 2024-09-13T00:26:29.244790+0000 mgr.y (mgr.14150) 249 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:31.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:30 smithi077 bash[32040]: cluster 2024-09-13T00:26:29.244790+0000 mgr.y (mgr.14150) 249 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:31.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:30 smithi088 bash[24895]: cluster 2024-09-13T00:26:29.244790+0000 mgr.y (mgr.14150) 249 : cluster [DBG] pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:33.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:32 smithi077 bash[22596]: cluster 2024-09-13T00:26:31.245345+0000 mgr.y (mgr.14150) 250 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:33.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:32 smithi077 bash[32040]: cluster 2024-09-13T00:26:31.245345+0000 mgr.y (mgr.14150) 250 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:33.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:32 smithi088 bash[24895]: cluster 2024-09-13T00:26:31.245345+0000 mgr.y (mgr.14150) 250 : cluster [DBG] pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:35.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:34 smithi077 bash[22596]: cluster 2024-09-13T00:26:33.245854+0000 mgr.y (mgr.14150) 251 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:35.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:34 smithi077 bash[32040]: cluster 2024-09-13T00:26:33.245854+0000 mgr.y (mgr.14150) 251 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:35.051 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:34 smithi088 bash[24895]: cluster 2024-09-13T00:26:33.245854+0000 mgr.y (mgr.14150) 251 : cluster [DBG] pgmap v222: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:36.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:36 smithi088 bash[24895]: cluster 2024-09-13T00:26:35.246395+0000 mgr.y (mgr.14150) 252 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:37.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:36 smithi077 bash[32040]: cluster 2024-09-13T00:26:35.246395+0000 mgr.y (mgr.14150) 252 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:37.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:36 smithi077 bash[22596]: cluster 2024-09-13T00:26:35.246395+0000 mgr.y (mgr.14150) 252 : cluster [DBG] pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:39.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:38 smithi077 bash[32040]: cluster 2024-09-13T00:26:37.246904+0000 mgr.y (mgr.14150) 253 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:39.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:38 smithi077 bash[32040]: audit 2024-09-13T00:26:38.168811+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T00:26:39.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:38 smithi077 bash[32040]: audit 2024-09-13T00:26:38.170868+0000 mon.a (mon.0) 483 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:39.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:38 smithi077 bash[22596]: cluster 2024-09-13T00:26:37.246904+0000 mgr.y (mgr.14150) 253 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:39.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:38 smithi077 bash[22596]: audit 2024-09-13T00:26:38.168811+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T00:26:39.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:38 smithi077 bash[22596]: audit 2024-09-13T00:26:38.170868+0000 mon.a (mon.0) 483 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:39.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:38 smithi088 bash[24895]: cluster 2024-09-13T00:26:37.246904+0000 mgr.y (mgr.14150) 253 : cluster [DBG] pgmap v224: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:39.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:38 smithi088 bash[24895]: audit 2024-09-13T00:26:38.168811+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T00:26:39.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:38 smithi088 bash[24895]: audit 2024-09-13T00:26:38.170868+0000 mon.a (mon.0) 483 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:40.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:39 smithi077 bash[22596]: cephadm 2024-09-13T00:26:38.172602+0000 mgr.y (mgr.14150) 254 : cephadm [INF] Deploying daemon osd.5 on smithi088 2024-09-13T00:26:40.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:39 smithi077 bash[32040]: cephadm 2024-09-13T00:26:38.172602+0000 mgr.y (mgr.14150) 254 : cephadm [INF] Deploying daemon osd.5 on smithi088 2024-09-13T00:26:40.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:39 smithi088 bash[24895]: cephadm 2024-09-13T00:26:38.172602+0000 mgr.y (mgr.14150) 254 : cephadm [INF] Deploying daemon osd.5 on smithi088 2024-09-13T00:26:41.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:40 smithi077 bash[22596]: cluster 2024-09-13T00:26:39.247369+0000 mgr.y (mgr.14150) 255 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:41.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:40 smithi077 bash[32040]: cluster 2024-09-13T00:26:39.247369+0000 mgr.y (mgr.14150) 255 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:41.097 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:40 smithi088 bash[24895]: cluster 2024-09-13T00:26:39.247369+0000 mgr.y (mgr.14150) 255 : cluster [DBG] pgmap v225: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:41.984 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:26:41 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:26:41.984 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:41 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:26:41.985 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:26:41 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:26:42.342 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:42 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:26:42.343 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:26:42 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:26:42.343 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:26:42 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:26:43.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:42 smithi077 bash[22596]: cluster 2024-09-13T00:26:41.247888+0000 mgr.y (mgr.14150) 256 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:43.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:42 smithi077 bash[22596]: audit 2024-09-13T00:26:42.263271+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:26:43.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:42 smithi077 bash[22596]: audit 2024-09-13T00:26:42.270891+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:43.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:42 smithi077 bash[22596]: audit 2024-09-13T00:26:42.278832+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:43.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:42 smithi077 bash[32040]: cluster 2024-09-13T00:26:41.247888+0000 mgr.y (mgr.14150) 256 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:43.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:42 smithi077 bash[32040]: audit 2024-09-13T00:26:42.263271+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:26:43.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:42 smithi077 bash[32040]: audit 2024-09-13T00:26:42.270891+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:43.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:42 smithi077 bash[32040]: audit 2024-09-13T00:26:42.278832+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:43.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:42 smithi088 bash[24895]: cluster 2024-09-13T00:26:41.247888+0000 mgr.y (mgr.14150) 256 : cluster [DBG] pgmap v226: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:43.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:42 smithi088 bash[24895]: audit 2024-09-13T00:26:42.263271+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:26:43.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:42 smithi088 bash[24895]: audit 2024-09-13T00:26:42.270891+0000 mon.a (mon.0) 485 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:43.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:42 smithi088 bash[24895]: audit 2024-09-13T00:26:42.278832+0000 mon.a (mon.0) 486 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:44.983 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:44 smithi088 bash[24895]: cluster 2024-09-13T00:26:43.248375+0000 mgr.y (mgr.14150) 257 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:45.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:44 smithi077 bash[22596]: cluster 2024-09-13T00:26:43.248375+0000 mgr.y (mgr.14150) 257 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:45.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:44 smithi077 bash[32040]: cluster 2024-09-13T00:26:43.248375+0000 mgr.y (mgr.14150) 257 : cluster [DBG] pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:47.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:46 smithi077 bash[32040]: cluster 2024-09-13T00:26:45.248850+0000 mgr.y (mgr.14150) 258 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:47.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:46 smithi077 bash[22596]: cluster 2024-09-13T00:26:45.248850+0000 mgr.y (mgr.14150) 258 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:47.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:46 smithi088 bash[24895]: cluster 2024-09-13T00:26:45.248850+0000 mgr.y (mgr.14150) 258 : cluster [DBG] pgmap v228: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:48.591 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:48 smithi088 bash[24895]: cluster 2024-09-13T00:26:47.249355+0000 mgr.y (mgr.14150) 259 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:48.591 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:48 smithi088 bash[24895]: audit 2024-09-13T00:26:47.337506+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:48.591 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:48 smithi088 bash[24895]: audit 2024-09-13T00:26:47.346139+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:48.591 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:48 smithi088 bash[24895]: audit 2024-09-13T00:26:47.347898+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:48.591 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:48 smithi088 bash[24895]: audit 2024-09-13T00:26:47.349552+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:26:48.591 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:48 smithi088 bash[24895]: audit 2024-09-13T00:26:47.358984+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:48.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:48 smithi077 bash[22596]: cluster 2024-09-13T00:26:47.249355+0000 mgr.y (mgr.14150) 259 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:48 smithi077 bash[22596]: audit 2024-09-13T00:26:47.337506+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:48 smithi077 bash[22596]: audit 2024-09-13T00:26:47.346139+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:48 smithi077 bash[22596]: audit 2024-09-13T00:26:47.347898+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:48 smithi077 bash[22596]: audit 2024-09-13T00:26:47.349552+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:26:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:48 smithi077 bash[22596]: audit 2024-09-13T00:26:47.358984+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:48.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:48 smithi077 bash[32040]: cluster 2024-09-13T00:26:47.249355+0000 mgr.y (mgr.14150) 259 : cluster [DBG] pgmap v229: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:48.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:48 smithi077 bash[32040]: audit 2024-09-13T00:26:47.337506+0000 mon.a (mon.0) 487 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:48.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:48 smithi077 bash[32040]: audit 2024-09-13T00:26:47.346139+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:48.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:48 smithi077 bash[32040]: audit 2024-09-13T00:26:47.347898+0000 mon.a (mon.0) 489 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:26:48.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:48 smithi077 bash[32040]: audit 2024-09-13T00:26:47.349552+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:26:48.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:48 smithi077 bash[32040]: audit 2024-09-13T00:26:47.358984+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:49.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:49 smithi088 bash[24895]: audit 2024-09-13T00:26:48.596831+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.88:6808/2699666173,v1:172.21.15.88:6809/2699666173]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T00:26:49.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:49 smithi088 bash[24895]: audit 2024-09-13T00:26:48.597843+0000 mon.a (mon.0) 492 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T00:26:49.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:49 smithi077 bash[32040]: audit 2024-09-13T00:26:48.596831+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.88:6808/2699666173,v1:172.21.15.88:6809/2699666173]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T00:26:49.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:49 smithi077 bash[32040]: audit 2024-09-13T00:26:48.597843+0000 mon.a (mon.0) 492 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T00:26:49.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:49 smithi077 bash[22596]: audit 2024-09-13T00:26:48.596831+0000 mon.b (mon.1) 15 : audit [INF] from='osd.5 [v2:172.21.15.88:6808/2699666173,v1:172.21.15.88:6809/2699666173]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T00:26:49.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:49 smithi077 bash[22596]: audit 2024-09-13T00:26:48.597843+0000 mon.a (mon.0) 492 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-13T00:26:50.450 INFO:teuthology.orchestra.run.smithi088.stdout:Created osd(s) 5 on host 'smithi088' 2024-09-13T00:26:50.667 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:50 smithi088 bash[24895]: cluster 2024-09-13T00:26:49.249853+0000 mgr.y (mgr.14150) 260 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:50.667 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:50 smithi088 bash[24895]: audit 2024-09-13T00:26:49.348042+0000 mon.a (mon.0) 493 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T00:26:50.667 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:50 smithi088 bash[24895]: cluster 2024-09-13T00:26:49.348158+0000 mon.a (mon.0) 494 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2024-09-13T00:26:50.667 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:50 smithi088 bash[24895]: audit 2024-09-13T00:26:49.348423+0000 mon.a (mon.0) 495 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:26:50.668 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:50 smithi088 bash[24895]: audit 2024-09-13T00:26:49.356131+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.88:6808/2699666173,v1:172.21.15.88:6809/2699666173]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:26:50.668 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:50 smithi088 bash[24895]: audit 2024-09-13T00:26:49.357448+0000 mon.a (mon.0) 496 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:26:50.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:50 smithi077 bash[22596]: cluster 2024-09-13T00:26:49.249853+0000 mgr.y (mgr.14150) 260 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:50.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:50 smithi077 bash[22596]: audit 2024-09-13T00:26:49.348042+0000 mon.a (mon.0) 493 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T00:26:50.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:50 smithi077 bash[22596]: cluster 2024-09-13T00:26:49.348158+0000 mon.a (mon.0) 494 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2024-09-13T00:26:50.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:50 smithi077 bash[22596]: audit 2024-09-13T00:26:49.348423+0000 mon.a (mon.0) 495 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:26:50.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:50 smithi077 bash[22596]: audit 2024-09-13T00:26:49.356131+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.88:6808/2699666173,v1:172.21.15.88:6809/2699666173]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:26:50.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:50 smithi077 bash[22596]: audit 2024-09-13T00:26:49.357448+0000 mon.a (mon.0) 496 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:26:50.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:50 smithi077 bash[32040]: cluster 2024-09-13T00:26:49.249853+0000 mgr.y (mgr.14150) 260 : cluster [DBG] pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:50.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:50 smithi077 bash[32040]: audit 2024-09-13T00:26:49.348042+0000 mon.a (mon.0) 493 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-13T00:26:50.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:50 smithi077 bash[32040]: cluster 2024-09-13T00:26:49.348158+0000 mon.a (mon.0) 494 : cluster [DBG] osdmap e34: 6 total, 5 up, 6 in 2024-09-13T00:26:50.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:50 smithi077 bash[32040]: audit 2024-09-13T00:26:49.348423+0000 mon.a (mon.0) 495 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:26:50.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:50 smithi077 bash[32040]: audit 2024-09-13T00:26:49.356131+0000 mon.b (mon.1) 16 : audit [INF] from='osd.5 [v2:172.21.15.88:6808/2699666173,v1:172.21.15.88:6809/2699666173]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:26:50.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:50 smithi077 bash[32040]: audit 2024-09-13T00:26:49.357448+0000 mon.a (mon.0) 496 : audit [INF] from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:26:51.026 DEBUG:teuthology.orchestra.run.smithi088:osd.5> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.5.service 2024-09-13T00:26:51.029 INFO:tasks.cephadm:Deploying osd.6 on smithi088 with /dev/vg_nvme/lv_2... 2024-09-13T00:26:51.029 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-09-13T00:26:51.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[22596]: audit 2024-09-13T00:26:50.350973+0000 mon.a (mon.0) 497 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-13T00:26:51.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[22596]: cluster 2024-09-13T00:26:50.351120+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-09-13T00:26:51.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[22596]: audit 2024-09-13T00:26:50.351958+0000 mon.a (mon.0) 499 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:26:51.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[22596]: audit 2024-09-13T00:26:50.361640+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:26:51.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[22596]: audit 2024-09-13T00:26:50.431735+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:26:51.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[22596]: audit 2024-09-13T00:26:50.439280+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:51.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[22596]: audit 2024-09-13T00:26:50.448036+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:51.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[22596]: cluster 2024-09-13T00:26:50.790336+0000 mon.a (mon.0) 504 : cluster [INF] osd.5 [v2:172.21.15.88:6808/2699666173,v1:172.21.15.88:6809/2699666173] boot 2024-09-13T00:26:51.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[22596]: cluster 2024-09-13T00:26:50.790440+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2024-09-13T00:26:51.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[22596]: audit 2024-09-13T00:26:50.790761+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:26:51.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[32040]: audit 2024-09-13T00:26:50.350973+0000 mon.a (mon.0) 497 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-13T00:26:51.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[32040]: cluster 2024-09-13T00:26:50.351120+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-09-13T00:26:51.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[32040]: audit 2024-09-13T00:26:50.351958+0000 mon.a (mon.0) 499 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:26:51.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[32040]: audit 2024-09-13T00:26:50.361640+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:26:51.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[32040]: audit 2024-09-13T00:26:50.431735+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:26:51.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[32040]: audit 2024-09-13T00:26:50.439280+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:51.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[32040]: audit 2024-09-13T00:26:50.448036+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:51.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[32040]: cluster 2024-09-13T00:26:50.790336+0000 mon.a (mon.0) 504 : cluster [INF] osd.5 [v2:172.21.15.88:6808/2699666173,v1:172.21.15.88:6809/2699666173] boot 2024-09-13T00:26:51.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[32040]: cluster 2024-09-13T00:26:50.790440+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2024-09-13T00:26:51.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:51 smithi077 bash[32040]: audit 2024-09-13T00:26:50.790761+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:26:51.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:51 smithi088 bash[24895]: audit 2024-09-13T00:26:50.350973+0000 mon.a (mon.0) 497 : audit [INF] from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-13T00:26:51.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:51 smithi088 bash[24895]: cluster 2024-09-13T00:26:50.351120+0000 mon.a (mon.0) 498 : cluster [DBG] osdmap e35: 6 total, 5 up, 6 in 2024-09-13T00:26:51.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:51 smithi088 bash[24895]: audit 2024-09-13T00:26:50.351958+0000 mon.a (mon.0) 499 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:26:51.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:51 smithi088 bash[24895]: audit 2024-09-13T00:26:50.361640+0000 mon.a (mon.0) 500 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:26:51.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:51 smithi088 bash[24895]: audit 2024-09-13T00:26:50.431735+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:26:51.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:51 smithi088 bash[24895]: audit 2024-09-13T00:26:50.439280+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:51.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:51 smithi088 bash[24895]: audit 2024-09-13T00:26:50.448036+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:26:51.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:51 smithi088 bash[24895]: cluster 2024-09-13T00:26:50.790336+0000 mon.a (mon.0) 504 : cluster [INF] osd.5 [v2:172.21.15.88:6808/2699666173,v1:172.21.15.88:6809/2699666173] boot 2024-09-13T00:26:51.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:51 smithi088 bash[24895]: cluster 2024-09-13T00:26:50.790440+0000 mon.a (mon.0) 505 : cluster [DBG] osdmap e36: 6 total, 6 up, 6 in 2024-09-13T00:26:51.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:51 smithi088 bash[24895]: audit 2024-09-13T00:26:50.790761+0000 mon.a (mon.0) 506 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:26:52.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:52 smithi077 bash[22596]: cluster 2024-09-13T00:26:49.595908+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:26:52.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:52 smithi077 bash[22596]: cluster 2024-09-13T00:26:49.596014+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:26:52.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:52 smithi077 bash[22596]: cluster 2024-09-13T00:26:51.250362+0000 mgr.y (mgr.14150) 261 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:52.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:52 smithi077 bash[22596]: cluster 2024-09-13T00:26:51.797682+0000 mon.a (mon.0) 507 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2024-09-13T00:26:52.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:52 smithi077 bash[32040]: cluster 2024-09-13T00:26:49.595908+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:26:52.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:52 smithi077 bash[32040]: cluster 2024-09-13T00:26:49.596014+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:26:52.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:52 smithi077 bash[32040]: cluster 2024-09-13T00:26:51.250362+0000 mgr.y (mgr.14150) 261 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:52.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:52 smithi077 bash[32040]: cluster 2024-09-13T00:26:51.797682+0000 mon.a (mon.0) 507 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2024-09-13T00:26:52.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:52 smithi088 bash[24895]: cluster 2024-09-13T00:26:49.595908+0000 osd.5 (osd.5) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:26:52.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:52 smithi088 bash[24895]: cluster 2024-09-13T00:26:49.596014+0000 osd.5 (osd.5) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:26:52.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:52 smithi088 bash[24895]: cluster 2024-09-13T00:26:51.250362+0000 mgr.y (mgr.14150) 261 : cluster [DBG] pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-09-13T00:26:52.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:52 smithi088 bash[24895]: cluster 2024-09-13T00:26:51.797682+0000 mon.a (mon.0) 507 : cluster [DBG] osdmap e37: 6 total, 6 up, 6 in 2024-09-13T00:26:54.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:53 smithi088 bash[24895]: cluster 2024-09-13T00:26:52.800963+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-09-13T00:26:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:53 smithi077 bash[22596]: cluster 2024-09-13T00:26:52.800963+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-09-13T00:26:54.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:53 smithi077 bash[32040]: cluster 2024-09-13T00:26:52.800963+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e38: 6 total, 6 up, 6 in 2024-09-13T00:26:55.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:54 smithi088 bash[24895]: cluster 2024-09-13T00:26:53.250879+0000 mgr.y (mgr.14150) 262 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:26:55.208 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T00:26:55.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:54 smithi077 bash[22596]: cluster 2024-09-13T00:26:53.250879+0000 mgr.y (mgr.14150) 262 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:26:55.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:54 smithi077 bash[32040]: cluster 2024-09-13T00:26:53.250879+0000 mgr.y (mgr.14150) 262 : cluster [DBG] pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:26:56.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:56 smithi088 bash[24895]: cluster 2024-09-13T00:26:55.251393+0000 mgr.y (mgr.14150) 263 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:26:57.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:56 smithi077 bash[22596]: cluster 2024-09-13T00:26:55.251393+0000 mgr.y (mgr.14150) 263 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:26:57.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:56 smithi077 bash[32040]: cluster 2024-09-13T00:26:55.251393+0000 mgr.y (mgr.14150) 263 : cluster [DBG] pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:26:59.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:26:58 smithi088 bash[24895]: cluster 2024-09-13T00:26:57.251900+0000 mgr.y (mgr.14150) 264 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:26:59.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:26:58 smithi077 bash[22596]: cluster 2024-09-13T00:26:57.251900+0000 mgr.y (mgr.14150) 264 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:26:59.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:26:58 smithi077 bash[32040]: cluster 2024-09-13T00:26:57.251900+0000 mgr.y (mgr.14150) 264 : cluster [DBG] pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:26:59.295 INFO:teuthology.orchestra.run.smithi088.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-09-13T00:26:59.505 INFO:teuthology.orchestra.run.smithi088.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-09-13T00:26:59.521 INFO:teuthology.orchestra.run.smithi088.stderr: stderr: 10+0 records in 2024-09-13T00:26:59.521 INFO:teuthology.orchestra.run.smithi088.stderr:10+0 records out 2024-09-13T00:26:59.521 INFO:teuthology.orchestra.run.smithi088.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0136595 s, 768 MB/s 2024-09-13T00:26:59.521 INFO:teuthology.orchestra.run.smithi088.stderr:--> Zapping successful for: 2024-09-13T00:27:00.172 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi088:vg_nvme/lv_2 2024-09-13T00:27:01.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:00 smithi088 bash[24895]: cluster 2024-09-13T00:26:59.252412+0000 mgr.y (mgr.14150) 265 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:01.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:00 smithi088 bash[24895]: audit 2024-09-13T00:27:00.464863+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:01.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:00 smithi088 bash[24895]: audit 2024-09-13T00:27:00.473926+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:01.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:00 smithi088 bash[24895]: audit 2024-09-13T00:27:00.476119+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:27:01.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:00 smithi088 bash[24895]: audit 2024-09-13T00:27:00.478047+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:27:01.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:00 smithi088 bash[24895]: audit 2024-09-13T00:27:00.487929+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:01.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:00 smithi088 bash[24895]: audit 2024-09-13T00:27:00.490450+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:01.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:00 smithi088 bash[24895]: audit 2024-09-13T00:27:00.492284+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:27:01.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:00 smithi088 bash[24895]: audit 2024-09-13T00:27:00.502571+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:01.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[22596]: cluster 2024-09-13T00:26:59.252412+0000 mgr.y (mgr.14150) 265 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[22596]: audit 2024-09-13T00:27:00.464863+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[22596]: audit 2024-09-13T00:27:00.473926+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[22596]: audit 2024-09-13T00:27:00.476119+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:27:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[22596]: audit 2024-09-13T00:27:00.478047+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:27:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[22596]: audit 2024-09-13T00:27:00.487929+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[22596]: audit 2024-09-13T00:27:00.490450+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[22596]: audit 2024-09-13T00:27:00.492284+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:27:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[22596]: audit 2024-09-13T00:27:00.502571+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:01.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[32040]: cluster 2024-09-13T00:26:59.252412+0000 mgr.y (mgr.14150) 265 : cluster [DBG] pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:01.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[32040]: audit 2024-09-13T00:27:00.464863+0000 mon.a (mon.0) 509 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:01.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[32040]: audit 2024-09-13T00:27:00.473926+0000 mon.a (mon.0) 510 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:01.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[32040]: audit 2024-09-13T00:27:00.476119+0000 mon.a (mon.0) 511 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:27:01.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[32040]: audit 2024-09-13T00:27:00.478047+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:27:01.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[32040]: audit 2024-09-13T00:27:00.487929+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:01.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[32040]: audit 2024-09-13T00:27:00.490450+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:01.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[32040]: audit 2024-09-13T00:27:00.492284+0000 mon.a (mon.0) 515 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:27:01.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:00 smithi077 bash[32040]: audit 2024-09-13T00:27:00.502571+0000 mon.a (mon.0) 516 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:02.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:01 smithi088 bash[24895]: cephadm 2024-09-13T00:27:00.455788+0000 mgr.y (mgr.14150) 266 : cephadm [INF] Detected new or changed devices on smithi088 2024-09-13T00:27:02.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:01 smithi088 bash[24895]: cephadm 2024-09-13T00:27:00.479415+0000 mgr.y (mgr.14150) 267 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 8632M 2024-09-13T00:27:02.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:01 smithi077 bash[22596]: cephadm 2024-09-13T00:27:00.455788+0000 mgr.y (mgr.14150) 266 : cephadm [INF] Detected new or changed devices on smithi088 2024-09-13T00:27:02.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:01 smithi077 bash[22596]: cephadm 2024-09-13T00:27:00.479415+0000 mgr.y (mgr.14150) 267 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 8632M 2024-09-13T00:27:02.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:01 smithi077 bash[32040]: cephadm 2024-09-13T00:27:00.455788+0000 mgr.y (mgr.14150) 266 : cephadm [INF] Detected new or changed devices on smithi088 2024-09-13T00:27:02.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:01 smithi077 bash[32040]: cephadm 2024-09-13T00:27:00.479415+0000 mgr.y (mgr.14150) 267 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 8632M 2024-09-13T00:27:03.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:02 smithi088 bash[24895]: cluster 2024-09-13T00:27:01.252910+0000 mgr.y (mgr.14150) 268 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:03.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:02 smithi077 bash[22596]: cluster 2024-09-13T00:27:01.252910+0000 mgr.y (mgr.14150) 268 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:03.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:02 smithi077 bash[32040]: cluster 2024-09-13T00:27:01.252910+0000 mgr.y (mgr.14150) 268 : cluster [DBG] pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:04.901 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T00:27:05.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:04 smithi088 bash[24895]: cluster 2024-09-13T00:27:03.253453+0000 mgr.y (mgr.14150) 269 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:05.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:04 smithi077 bash[22596]: cluster 2024-09-13T00:27:03.253453+0000 mgr.y (mgr.14150) 269 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:05.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:04 smithi077 bash[32040]: cluster 2024-09-13T00:27:03.253453+0000 mgr.y (mgr.14150) 269 : cluster [DBG] pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:07.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:06 smithi088 bash[24895]: cluster 2024-09-13T00:27:05.253952+0000 mgr.y (mgr.14150) 270 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:07.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:06 smithi077 bash[22596]: cluster 2024-09-13T00:27:05.253952+0000 mgr.y (mgr.14150) 270 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:07.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:06 smithi077 bash[32040]: cluster 2024-09-13T00:27:05.253952+0000 mgr.y (mgr.14150) 270 : cluster [DBG] pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:08.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:07 smithi088 bash[24895]: audit 2024-09-13T00:27:07.621147+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:27:08.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:07 smithi088 bash[24895]: audit 2024-09-13T00:27:07.627075+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:27:08.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:07 smithi088 bash[24895]: audit 2024-09-13T00:27:07.628676+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:08.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:07 smithi077 bash[22596]: audit 2024-09-13T00:27:07.621147+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:27:08.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:07 smithi077 bash[22596]: audit 2024-09-13T00:27:07.627075+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:27:08.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:07 smithi077 bash[22596]: audit 2024-09-13T00:27:07.628676+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:08.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:07 smithi077 bash[32040]: audit 2024-09-13T00:27:07.621147+0000 mon.a (mon.0) 517 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:27:08.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:07 smithi077 bash[32040]: audit 2024-09-13T00:27:07.627075+0000 mon.a (mon.0) 518 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:27:08.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:07 smithi077 bash[32040]: audit 2024-09-13T00:27:07.628676+0000 mon.a (mon.0) 519 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:09.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:08 smithi088 bash[24895]: cluster 2024-09-13T00:27:07.254588+0000 mgr.y (mgr.14150) 271 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:09.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:08 smithi088 bash[24895]: audit 2024-09-13T00:27:07.614832+0000 mgr.y (mgr.14150) 272 : audit [DBG] from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:27:09.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:08 smithi077 bash[22596]: cluster 2024-09-13T00:27:07.254588+0000 mgr.y (mgr.14150) 271 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:09.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:08 smithi077 bash[22596]: audit 2024-09-13T00:27:07.614832+0000 mgr.y (mgr.14150) 272 : audit [DBG] from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:27:09.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:08 smithi077 bash[32040]: cluster 2024-09-13T00:27:07.254588+0000 mgr.y (mgr.14150) 271 : cluster [DBG] pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:09.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:08 smithi077 bash[32040]: audit 2024-09-13T00:27:07.614832+0000 mgr.y (mgr.14150) 272 : audit [DBG] from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:27:11.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:10 smithi088 bash[24895]: cluster 2024-09-13T00:27:09.255161+0000 mgr.y (mgr.14150) 273 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:11.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:10 smithi077 bash[22596]: cluster 2024-09-13T00:27:09.255161+0000 mgr.y (mgr.14150) 273 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:11.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:10 smithi077 bash[32040]: cluster 2024-09-13T00:27:09.255161+0000 mgr.y (mgr.14150) 273 : cluster [DBG] pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:13.241 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:12 smithi088 bash[24895]: cluster 2024-09-13T00:27:11.255672+0000 mgr.y (mgr.14150) 274 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:13.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:12 smithi077 bash[22596]: cluster 2024-09-13T00:27:11.255672+0000 mgr.y (mgr.14150) 274 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:13.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:12 smithi077 bash[32040]: cluster 2024-09-13T00:27:11.255672+0000 mgr.y (mgr.14150) 274 : cluster [DBG] pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:15.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:14 smithi088 bash[24895]: cluster 2024-09-13T00:27:13.256194+0000 mgr.y (mgr.14150) 275 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:15.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:14 smithi077 bash[32040]: cluster 2024-09-13T00:27:13.256194+0000 mgr.y (mgr.14150) 275 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:15.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:14 smithi077 bash[22596]: cluster 2024-09-13T00:27:13.256194+0000 mgr.y (mgr.14150) 275 : cluster [DBG] pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:16.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:15 smithi077 bash[22596]: audit 2024-09-13T00:27:15.514248+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.88:0/369240448' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "31c0d169-c329-440d-848b-057fbbf418ca"}]: dispatch 2024-09-13T00:27:16.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:15 smithi077 bash[22596]: audit 2024-09-13T00:27:15.515386+0000 mon.a (mon.0) 520 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "31c0d169-c329-440d-848b-057fbbf418ca"}]: dispatch 2024-09-13T00:27:16.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:15 smithi077 bash[22596]: audit 2024-09-13T00:27:15.526082+0000 mon.a (mon.0) 521 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "31c0d169-c329-440d-848b-057fbbf418ca"}]': finished 2024-09-13T00:27:16.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:15 smithi077 bash[22596]: cluster 2024-09-13T00:27:15.526228+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2024-09-13T00:27:16.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:15 smithi077 bash[22596]: audit 2024-09-13T00:27:15.526623+0000 mon.a (mon.0) 523 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:27:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:15 smithi077 bash[32040]: audit 2024-09-13T00:27:15.514248+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.88:0/369240448' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "31c0d169-c329-440d-848b-057fbbf418ca"}]: dispatch 2024-09-13T00:27:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:15 smithi077 bash[32040]: audit 2024-09-13T00:27:15.515386+0000 mon.a (mon.0) 520 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "31c0d169-c329-440d-848b-057fbbf418ca"}]: dispatch 2024-09-13T00:27:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:15 smithi077 bash[32040]: audit 2024-09-13T00:27:15.526082+0000 mon.a (mon.0) 521 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "31c0d169-c329-440d-848b-057fbbf418ca"}]': finished 2024-09-13T00:27:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:15 smithi077 bash[32040]: cluster 2024-09-13T00:27:15.526228+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2024-09-13T00:27:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:15 smithi077 bash[32040]: audit 2024-09-13T00:27:15.526623+0000 mon.a (mon.0) 523 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:27:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:15 smithi088 bash[24895]: audit 2024-09-13T00:27:15.514248+0000 mon.b (mon.1) 17 : audit [INF] from='client.? 172.21.15.88:0/369240448' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "31c0d169-c329-440d-848b-057fbbf418ca"}]: dispatch 2024-09-13T00:27:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:15 smithi088 bash[24895]: audit 2024-09-13T00:27:15.515386+0000 mon.a (mon.0) 520 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "31c0d169-c329-440d-848b-057fbbf418ca"}]: dispatch 2024-09-13T00:27:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:15 smithi088 bash[24895]: audit 2024-09-13T00:27:15.526082+0000 mon.a (mon.0) 521 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "31c0d169-c329-440d-848b-057fbbf418ca"}]': finished 2024-09-13T00:27:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:15 smithi088 bash[24895]: cluster 2024-09-13T00:27:15.526228+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e39: 7 total, 6 up, 7 in 2024-09-13T00:27:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:15 smithi088 bash[24895]: audit 2024-09-13T00:27:15.526623+0000 mon.a (mon.0) 523 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:27:17.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:16 smithi077 bash[32040]: cluster 2024-09-13T00:27:15.256709+0000 mgr.y (mgr.14150) 276 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:17.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:16 smithi077 bash[32040]: audit 2024-09-13T00:27:16.144238+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.88:0/1472747573' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:27:17.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:16 smithi077 bash[22596]: cluster 2024-09-13T00:27:15.256709+0000 mgr.y (mgr.14150) 276 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:17.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:16 smithi077 bash[22596]: audit 2024-09-13T00:27:16.144238+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.88:0/1472747573' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:27:17.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:16 smithi088 bash[24895]: cluster 2024-09-13T00:27:15.256709+0000 mgr.y (mgr.14150) 276 : cluster [DBG] pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:17.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:16 smithi088 bash[24895]: audit 2024-09-13T00:27:16.144238+0000 mon.b (mon.1) 18 : audit [DBG] from='client.? 172.21.15.88:0/1472747573' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:27:19.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:18 smithi077 bash[22596]: cluster 2024-09-13T00:27:17.257226+0000 mgr.y (mgr.14150) 277 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:19.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:18 smithi077 bash[32040]: cluster 2024-09-13T00:27:17.257226+0000 mgr.y (mgr.14150) 277 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:19.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:18 smithi088 bash[24895]: cluster 2024-09-13T00:27:17.257226+0000 mgr.y (mgr.14150) 277 : cluster [DBG] pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:21.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:20 smithi077 bash[22596]: cluster 2024-09-13T00:27:19.257769+0000 mgr.y (mgr.14150) 278 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:21.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:20 smithi077 bash[32040]: cluster 2024-09-13T00:27:19.257769+0000 mgr.y (mgr.14150) 278 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:21.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:20 smithi088 bash[24895]: cluster 2024-09-13T00:27:19.257769+0000 mgr.y (mgr.14150) 278 : cluster [DBG] pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:23.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:22 smithi077 bash[22596]: cluster 2024-09-13T00:27:21.258281+0000 mgr.y (mgr.14150) 279 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:23.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:22 smithi077 bash[32040]: cluster 2024-09-13T00:27:21.258281+0000 mgr.y (mgr.14150) 279 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:23.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:22 smithi088 bash[24895]: cluster 2024-09-13T00:27:21.258281+0000 mgr.y (mgr.14150) 279 : cluster [DBG] pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:25.159 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:24 smithi088 bash[24895]: cluster 2024-09-13T00:27:23.258858+0000 mgr.y (mgr.14150) 280 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:25.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:24 smithi077 bash[22596]: cluster 2024-09-13T00:27:23.258858+0000 mgr.y (mgr.14150) 280 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:25.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:24 smithi077 bash[32040]: cluster 2024-09-13T00:27:23.258858+0000 mgr.y (mgr.14150) 280 : cluster [DBG] pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:27.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:26 smithi088 bash[24895]: cluster 2024-09-13T00:27:25.259374+0000 mgr.y (mgr.14150) 281 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:27.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:26 smithi077 bash[22596]: cluster 2024-09-13T00:27:25.259374+0000 mgr.y (mgr.14150) 281 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:27.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:26 smithi077 bash[32040]: cluster 2024-09-13T00:27:25.259374+0000 mgr.y (mgr.14150) 281 : cluster [DBG] pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:28.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:27 smithi077 bash[22596]: audit 2024-09-13T00:27:27.315712+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T00:27:28.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:27 smithi077 bash[22596]: audit 2024-09-13T00:27:27.317368+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:28.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:27 smithi077 bash[32040]: audit 2024-09-13T00:27:27.315712+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T00:27:28.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:27 smithi077 bash[32040]: audit 2024-09-13T00:27:27.317368+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:27 smithi088 bash[24895]: audit 2024-09-13T00:27:27.315712+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T00:27:28.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:27 smithi088 bash[24895]: audit 2024-09-13T00:27:27.317368+0000 mon.a (mon.0) 525 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:29.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:28 smithi088 bash[24895]: cluster 2024-09-13T00:27:27.259900+0000 mgr.y (mgr.14150) 282 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:29.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:28 smithi088 bash[24895]: cephadm 2024-09-13T00:27:27.318895+0000 mgr.y (mgr.14150) 283 : cephadm [INF] Deploying daemon osd.6 on smithi088 2024-09-13T00:27:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:28 smithi077 bash[22596]: cluster 2024-09-13T00:27:27.259900+0000 mgr.y (mgr.14150) 282 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:28 smithi077 bash[22596]: cephadm 2024-09-13T00:27:27.318895+0000 mgr.y (mgr.14150) 283 : cephadm [INF] Deploying daemon osd.6 on smithi088 2024-09-13T00:27:29.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:28 smithi077 bash[32040]: cluster 2024-09-13T00:27:27.259900+0000 mgr.y (mgr.14150) 282 : cluster [DBG] pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:29.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:28 smithi077 bash[32040]: cephadm 2024-09-13T00:27:27.318895+0000 mgr.y (mgr.14150) 283 : cephadm [INF] Deploying daemon osd.6 on smithi088 2024-09-13T00:27:31.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:30 smithi077 bash[22596]: cluster 2024-09-13T00:27:29.260430+0000 mgr.y (mgr.14150) 284 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:31.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:30 smithi077 bash[32040]: cluster 2024-09-13T00:27:29.260430+0000 mgr.y (mgr.14150) 284 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:31.280 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:27:31 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:27:31.281 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:27:31 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:27:31.281 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:30 smithi088 bash[24895]: cluster 2024-09-13T00:27:29.260430+0000 mgr.y (mgr.14150) 284 : cluster [DBG] pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:31.281 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:31 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:27:31.281 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:27:31 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:27:31.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:31 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:27:31.601 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:27:31 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:27:31.602 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:27:31 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:27:31.602 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:27:31 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:27:32.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:31 smithi077 bash[22596]: audit 2024-09-13T00:27:31.417511+0000 mon.a (mon.0) 526 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:27:32.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:31 smithi077 bash[22596]: audit 2024-09-13T00:27:31.425268+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:32.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:31 smithi077 bash[22596]: audit 2024-09-13T00:27:31.433528+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:32.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:31 smithi077 bash[32040]: audit 2024-09-13T00:27:31.417511+0000 mon.a (mon.0) 526 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:27:32.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:31 smithi077 bash[32040]: audit 2024-09-13T00:27:31.425268+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:32.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:31 smithi077 bash[32040]: audit 2024-09-13T00:27:31.433528+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:32.339 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:31 smithi088 bash[24895]: audit 2024-09-13T00:27:31.417511+0000 mon.a (mon.0) 526 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:27:32.339 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:31 smithi088 bash[24895]: audit 2024-09-13T00:27:31.425268+0000 mon.a (mon.0) 527 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:32.339 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:31 smithi088 bash[24895]: audit 2024-09-13T00:27:31.433528+0000 mon.a (mon.0) 528 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:33.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:32 smithi088 bash[24895]: cluster 2024-09-13T00:27:31.260987+0000 mgr.y (mgr.14150) 285 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:33.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:32 smithi077 bash[22596]: cluster 2024-09-13T00:27:31.260987+0000 mgr.y (mgr.14150) 285 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:33.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:32 smithi077 bash[32040]: cluster 2024-09-13T00:27:31.260987+0000 mgr.y (mgr.14150) 285 : cluster [DBG] pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:34.221 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:33 smithi088 bash[24895]: cluster 2024-09-13T00:27:33.261647+0000 mgr.y (mgr.14150) 286 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:34.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:33 smithi077 bash[22596]: cluster 2024-09-13T00:27:33.261647+0000 mgr.y (mgr.14150) 286 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:34.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:33 smithi077 bash[32040]: cluster 2024-09-13T00:27:33.261647+0000 mgr.y (mgr.14150) 286 : cluster [DBG] pgmap v258: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:36.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:36 smithi088 bash[24895]: cluster 2024-09-13T00:27:35.262211+0000 mgr.y (mgr.14150) 287 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:36.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:36 smithi077 bash[22596]: cluster 2024-09-13T00:27:35.262211+0000 mgr.y (mgr.14150) 287 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:36.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:36 smithi077 bash[32040]: cluster 2024-09-13T00:27:35.262211+0000 mgr.y (mgr.14150) 287 : cluster [DBG] pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:37.728 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:37 smithi088 bash[24895]: audit 2024-09-13T00:27:36.514045+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:37.728 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:37 smithi088 bash[24895]: audit 2024-09-13T00:27:36.523121+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:37.728 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:37 smithi088 bash[24895]: audit 2024-09-13T00:27:36.941794+0000 mon.a (mon.0) 531 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:37.728 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:37 smithi088 bash[24895]: audit 2024-09-13T00:27:36.943332+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:27:37.728 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:37 smithi088 bash[24895]: audit 2024-09-13T00:27:36.952975+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:37.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:37 smithi077 bash[22596]: audit 2024-09-13T00:27:36.514045+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:37.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:37 smithi077 bash[22596]: audit 2024-09-13T00:27:36.523121+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:37.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:37 smithi077 bash[22596]: audit 2024-09-13T00:27:36.941794+0000 mon.a (mon.0) 531 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:37.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:37 smithi077 bash[22596]: audit 2024-09-13T00:27:36.943332+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:27:37.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:37 smithi077 bash[22596]: audit 2024-09-13T00:27:36.952975+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:37.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:37 smithi077 bash[32040]: audit 2024-09-13T00:27:36.514045+0000 mon.a (mon.0) 529 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:37.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:37 smithi077 bash[32040]: audit 2024-09-13T00:27:36.523121+0000 mon.a (mon.0) 530 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:37.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:37 smithi077 bash[32040]: audit 2024-09-13T00:27:36.941794+0000 mon.a (mon.0) 531 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:37.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:37 smithi077 bash[32040]: audit 2024-09-13T00:27:36.943332+0000 mon.a (mon.0) 532 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:27:37.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:37 smithi077 bash[32040]: audit 2024-09-13T00:27:36.952975+0000 mon.a (mon.0) 533 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:38.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:38 smithi077 bash[22596]: cluster 2024-09-13T00:27:37.262770+0000 mgr.y (mgr.14150) 288 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:38.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:38 smithi077 bash[22596]: audit 2024-09-13T00:27:37.734673+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.88:6816/412460705,v1:172.21.15.88:6817/412460705]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T00:27:38.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:38 smithi077 bash[22596]: audit 2024-09-13T00:27:37.735855+0000 mon.a (mon.0) 534 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T00:27:38.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:38 smithi077 bash[32040]: cluster 2024-09-13T00:27:37.262770+0000 mgr.y (mgr.14150) 288 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:38.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:38 smithi077 bash[32040]: audit 2024-09-13T00:27:37.734673+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.88:6816/412460705,v1:172.21.15.88:6817/412460705]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T00:27:38.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:38 smithi077 bash[32040]: audit 2024-09-13T00:27:37.735855+0000 mon.a (mon.0) 534 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T00:27:38.795 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:38 smithi088 bash[24895]: cluster 2024-09-13T00:27:37.262770+0000 mgr.y (mgr.14150) 288 : cluster [DBG] pgmap v260: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:38.795 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:38 smithi088 bash[24895]: audit 2024-09-13T00:27:37.734673+0000 mon.b (mon.1) 19 : audit [INF] from='osd.6 [v2:172.21.15.88:6816/412460705,v1:172.21.15.88:6817/412460705]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T00:27:38.795 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:38 smithi088 bash[24895]: audit 2024-09-13T00:27:37.735855+0000 mon.a (mon.0) 534 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-13T00:27:39.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:39 smithi088 bash[24895]: audit 2024-09-13T00:27:38.519612+0000 mon.a (mon.0) 535 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T00:27:39.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:39 smithi088 bash[24895]: cluster 2024-09-13T00:27:38.519728+0000 mon.a (mon.0) 536 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2024-09-13T00:27:39.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:39 smithi088 bash[24895]: audit 2024-09-13T00:27:38.520331+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:27:39.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:39 smithi088 bash[24895]: audit 2024-09-13T00:27:38.520420+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.88:6816/412460705,v1:172.21.15.88:6817/412460705]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:27:39.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:39 smithi088 bash[24895]: audit 2024-09-13T00:27:38.527608+0000 mon.a (mon.0) 538 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:27:39.881 INFO:teuthology.orchestra.run.smithi088.stdout:Created osd(s) 6 on host 'smithi088' 2024-09-13T00:27:40.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:39 smithi077 bash[22596]: audit 2024-09-13T00:27:38.519612+0000 mon.a (mon.0) 535 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T00:27:40.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:39 smithi077 bash[22596]: cluster 2024-09-13T00:27:38.519728+0000 mon.a (mon.0) 536 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2024-09-13T00:27:40.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:39 smithi077 bash[22596]: audit 2024-09-13T00:27:38.520331+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:27:40.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:39 smithi077 bash[22596]: audit 2024-09-13T00:27:38.520420+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.88:6816/412460705,v1:172.21.15.88:6817/412460705]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:27:40.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:39 smithi077 bash[22596]: audit 2024-09-13T00:27:38.527608+0000 mon.a (mon.0) 538 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:27:40.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:39 smithi077 bash[32040]: audit 2024-09-13T00:27:38.519612+0000 mon.a (mon.0) 535 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-13T00:27:40.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:39 smithi077 bash[32040]: cluster 2024-09-13T00:27:38.519728+0000 mon.a (mon.0) 536 : cluster [DBG] osdmap e40: 7 total, 6 up, 7 in 2024-09-13T00:27:40.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:39 smithi077 bash[32040]: audit 2024-09-13T00:27:38.520331+0000 mon.a (mon.0) 537 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:27:40.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:39 smithi077 bash[32040]: audit 2024-09-13T00:27:38.520420+0000 mon.b (mon.1) 20 : audit [INF] from='osd.6 [v2:172.21.15.88:6816/412460705,v1:172.21.15.88:6817/412460705]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:27:40.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:39 smithi077 bash[32040]: audit 2024-09-13T00:27:38.527608+0000 mon.a (mon.0) 538 : audit [INF] from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:27:40.510 DEBUG:teuthology.orchestra.run.smithi088:osd.6> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.6.service 2024-09-13T00:27:40.513 INFO:tasks.cephadm:Deploying osd.7 on smithi088 with /dev/vg_nvme/lv_1... 2024-09-13T00:27:40.513 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-09-13T00:27:40.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:40 smithi088 bash[24895]: cluster 2024-09-13T00:27:39.263246+0000 mgr.y (mgr.14150) 289 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:40.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:40 smithi088 bash[24895]: audit 2024-09-13T00:27:39.529906+0000 mon.a (mon.0) 539 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-13T00:27:40.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:40 smithi088 bash[24895]: cluster 2024-09-13T00:27:39.530025+0000 mon.a (mon.0) 540 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2024-09-13T00:27:40.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:40 smithi088 bash[24895]: audit 2024-09-13T00:27:39.530833+0000 mon.a (mon.0) 541 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:27:40.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:40 smithi088 bash[24895]: audit 2024-09-13T00:27:39.541091+0000 mon.a (mon.0) 542 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:27:40.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:40 smithi088 bash[24895]: audit 2024-09-13T00:27:39.862391+0000 mon.a (mon.0) 543 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:27:40.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:40 smithi088 bash[24895]: audit 2024-09-13T00:27:39.870828+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:40.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:40 smithi088 bash[24895]: audit 2024-09-13T00:27:39.879641+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:41.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:40 smithi077 bash[22596]: cluster 2024-09-13T00:27:39.263246+0000 mgr.y (mgr.14150) 289 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:41.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:40 smithi077 bash[22596]: audit 2024-09-13T00:27:39.529906+0000 mon.a (mon.0) 539 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-13T00:27:41.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:40 smithi077 bash[22596]: cluster 2024-09-13T00:27:39.530025+0000 mon.a (mon.0) 540 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2024-09-13T00:27:41.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:40 smithi077 bash[22596]: audit 2024-09-13T00:27:39.530833+0000 mon.a (mon.0) 541 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:27:41.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:40 smithi077 bash[22596]: audit 2024-09-13T00:27:39.541091+0000 mon.a (mon.0) 542 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:27:41.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:40 smithi077 bash[22596]: audit 2024-09-13T00:27:39.862391+0000 mon.a (mon.0) 543 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:27:41.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:40 smithi077 bash[22596]: audit 2024-09-13T00:27:39.870828+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:41.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:40 smithi077 bash[22596]: audit 2024-09-13T00:27:39.879641+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:41.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:40 smithi077 bash[32040]: cluster 2024-09-13T00:27:39.263246+0000 mgr.y (mgr.14150) 289 : cluster [DBG] pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-09-13T00:27:41.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:40 smithi077 bash[32040]: audit 2024-09-13T00:27:39.529906+0000 mon.a (mon.0) 539 : audit [INF] from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-13T00:27:41.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:40 smithi077 bash[32040]: cluster 2024-09-13T00:27:39.530025+0000 mon.a (mon.0) 540 : cluster [DBG] osdmap e41: 7 total, 6 up, 7 in 2024-09-13T00:27:41.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:40 smithi077 bash[32040]: audit 2024-09-13T00:27:39.530833+0000 mon.a (mon.0) 541 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:27:41.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:40 smithi077 bash[32040]: audit 2024-09-13T00:27:39.541091+0000 mon.a (mon.0) 542 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:27:41.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:40 smithi077 bash[32040]: audit 2024-09-13T00:27:39.862391+0000 mon.a (mon.0) 543 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:27:41.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:40 smithi077 bash[32040]: audit 2024-09-13T00:27:39.870828+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:41.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:40 smithi077 bash[32040]: audit 2024-09-13T00:27:39.879641+0000 mon.a (mon.0) 545 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:41.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:41 smithi088 bash[24895]: cluster 2024-09-13T00:27:38.694465+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:27:41.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:41 smithi088 bash[24895]: cluster 2024-09-13T00:27:38.694571+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:27:41.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:41 smithi088 bash[24895]: cluster 2024-09-13T00:27:40.534014+0000 mon.a (mon.0) 546 : cluster [INF] osd.6 [v2:172.21.15.88:6816/412460705,v1:172.21.15.88:6817/412460705] boot 2024-09-13T00:27:41.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:41 smithi088 bash[24895]: cluster 2024-09-13T00:27:40.534103+0000 mon.a (mon.0) 547 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2024-09-13T00:27:41.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:41 smithi088 bash[24895]: audit 2024-09-13T00:27:40.534422+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:27:41.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:41 smithi088 bash[24895]: cluster 2024-09-13T00:27:40.803363+0000 mon.a (mon.0) 549 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2024-09-13T00:27:42.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:41 smithi077 bash[22596]: cluster 2024-09-13T00:27:38.694465+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:27:42.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:41 smithi077 bash[22596]: cluster 2024-09-13T00:27:38.694571+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:27:42.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:41 smithi077 bash[22596]: cluster 2024-09-13T00:27:40.534014+0000 mon.a (mon.0) 546 : cluster [INF] osd.6 [v2:172.21.15.88:6816/412460705,v1:172.21.15.88:6817/412460705] boot 2024-09-13T00:27:42.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:41 smithi077 bash[22596]: cluster 2024-09-13T00:27:40.534103+0000 mon.a (mon.0) 547 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2024-09-13T00:27:42.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:41 smithi077 bash[22596]: audit 2024-09-13T00:27:40.534422+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:27:42.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:41 smithi077 bash[22596]: cluster 2024-09-13T00:27:40.803363+0000 mon.a (mon.0) 549 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2024-09-13T00:27:42.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:41 smithi077 bash[32040]: cluster 2024-09-13T00:27:38.694465+0000 osd.6 (osd.6) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:27:42.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:41 smithi077 bash[32040]: cluster 2024-09-13T00:27:38.694571+0000 osd.6 (osd.6) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:27:42.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:41 smithi077 bash[32040]: cluster 2024-09-13T00:27:40.534014+0000 mon.a (mon.0) 546 : cluster [INF] osd.6 [v2:172.21.15.88:6816/412460705,v1:172.21.15.88:6817/412460705] boot 2024-09-13T00:27:42.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:41 smithi077 bash[32040]: cluster 2024-09-13T00:27:40.534103+0000 mon.a (mon.0) 547 : cluster [DBG] osdmap e42: 7 total, 7 up, 7 in 2024-09-13T00:27:42.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:41 smithi077 bash[32040]: audit 2024-09-13T00:27:40.534422+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:27:42.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:41 smithi077 bash[32040]: cluster 2024-09-13T00:27:40.803363+0000 mon.a (mon.0) 549 : cluster [DBG] osdmap e43: 7 total, 7 up, 7 in 2024-09-13T00:27:43.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:42 smithi088 bash[24895]: cluster 2024-09-13T00:27:41.263786+0000 mgr.y (mgr.14150) 290 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:43.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:42 smithi088 bash[24895]: cluster 2024-09-13T00:27:41.809927+0000 mon.a (mon.0) 550 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2024-09-13T00:27:43.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:42 smithi077 bash[22596]: cluster 2024-09-13T00:27:41.263786+0000 mgr.y (mgr.14150) 290 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:43.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:42 smithi077 bash[22596]: cluster 2024-09-13T00:27:41.809927+0000 mon.a (mon.0) 550 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2024-09-13T00:27:43.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:42 smithi077 bash[32040]: cluster 2024-09-13T00:27:41.263786+0000 mgr.y (mgr.14150) 290 : cluster [DBG] pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:43.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:42 smithi077 bash[32040]: cluster 2024-09-13T00:27:41.809927+0000 mon.a (mon.0) 550 : cluster [DBG] osdmap e44: 7 total, 7 up, 7 in 2024-09-13T00:27:44.639 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T00:27:45.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:44 smithi088 bash[24895]: cluster 2024-09-13T00:27:43.264313+0000 mgr.y (mgr.14150) 291 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:45.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:44 smithi077 bash[22596]: cluster 2024-09-13T00:27:43.264313+0000 mgr.y (mgr.14150) 291 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:45.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:44 smithi077 bash[32040]: cluster 2024-09-13T00:27:43.264313+0000 mgr.y (mgr.14150) 291 : cluster [DBG] pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:47.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:46 smithi088 bash[24895]: cluster 2024-09-13T00:27:45.264835+0000 mgr.y (mgr.14150) 292 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:47.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:46 smithi077 bash[22596]: cluster 2024-09-13T00:27:45.264835+0000 mgr.y (mgr.14150) 292 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:47.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:46 smithi077 bash[32040]: cluster 2024-09-13T00:27:45.264835+0000 mgr.y (mgr.14150) 292 : cluster [DBG] pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:48.846 INFO:teuthology.orchestra.run.smithi088.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-09-13T00:27:49.040 INFO:teuthology.orchestra.run.smithi088.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-09-13T00:27:49.056 INFO:teuthology.orchestra.run.smithi088.stderr: stderr: 10+0 records in 2024-09-13T00:27:49.056 INFO:teuthology.orchestra.run.smithi088.stderr:10+0 records out 2024-09-13T00:27:49.056 INFO:teuthology.orchestra.run.smithi088.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0135651 s, 773 MB/s 2024-09-13T00:27:49.056 INFO:teuthology.orchestra.run.smithi088.stderr:--> Zapping successful for: 2024-09-13T00:27:49.076 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:48 smithi088 bash[24895]: cluster 2024-09-13T00:27:47.265405+0000 mgr.y (mgr.14150) 293 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:49.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:48 smithi077 bash[32040]: cluster 2024-09-13T00:27:47.265405+0000 mgr.y (mgr.14150) 293 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:49.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:48 smithi077 bash[22596]: cluster 2024-09-13T00:27:47.265405+0000 mgr.y (mgr.14150) 293 : cluster [DBG] pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:49.567 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi088:vg_nvme/lv_1 2024-09-13T00:27:51.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:50 smithi088 bash[24895]: cluster 2024-09-13T00:27:49.265941+0000 mgr.y (mgr.14150) 294 : cluster [DBG] pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:51.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:50 smithi088 bash[24895]: audit 2024-09-13T00:27:50.039455+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:51.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:50 smithi088 bash[24895]: audit 2024-09-13T00:27:50.049194+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:51.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:50 smithi088 bash[24895]: audit 2024-09-13T00:27:50.051437+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:27:51.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:50 smithi088 bash[24895]: audit 2024-09-13T00:27:50.053358+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:27:51.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:50 smithi088 bash[24895]: audit 2024-09-13T00:27:50.055149+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:27:51.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:50 smithi088 bash[24895]: audit 2024-09-13T00:27:50.070125+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:51.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:50 smithi088 bash[24895]: audit 2024-09-13T00:27:50.072304+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:51.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:50 smithi088 bash[24895]: audit 2024-09-13T00:27:50.074387+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:27:51.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:50 smithi088 bash[24895]: audit 2024-09-13T00:27:50.088365+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:51.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[22596]: cluster 2024-09-13T00:27:49.265941+0000 mgr.y (mgr.14150) 294 : cluster [DBG] pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:51.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[22596]: audit 2024-09-13T00:27:50.039455+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:51.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[22596]: audit 2024-09-13T00:27:50.049194+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:51.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[22596]: audit 2024-09-13T00:27:50.051437+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:27:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[22596]: audit 2024-09-13T00:27:50.053358+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:27:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[22596]: audit 2024-09-13T00:27:50.055149+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:27:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[22596]: audit 2024-09-13T00:27:50.070125+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[22596]: audit 2024-09-13T00:27:50.072304+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[22596]: audit 2024-09-13T00:27:50.074387+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:27:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[22596]: audit 2024-09-13T00:27:50.088365+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:51.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[32040]: cluster 2024-09-13T00:27:49.265941+0000 mgr.y (mgr.14150) 294 : cluster [DBG] pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:51.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[32040]: audit 2024-09-13T00:27:50.039455+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:51.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[32040]: audit 2024-09-13T00:27:50.049194+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:51.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[32040]: audit 2024-09-13T00:27:50.051437+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:27:51.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[32040]: audit 2024-09-13T00:27:50.053358+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:27:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[32040]: audit 2024-09-13T00:27:50.055149+0000 mon.a (mon.0) 555 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:27:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[32040]: audit 2024-09-13T00:27:50.070125+0000 mon.a (mon.0) 556 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[32040]: audit 2024-09-13T00:27:50.072304+0000 mon.a (mon.0) 557 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[32040]: audit 2024-09-13T00:27:50.074387+0000 mon.a (mon.0) 558 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:27:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:50 smithi077 bash[32040]: audit 2024-09-13T00:27:50.088365+0000 mon.a (mon.0) 559 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:27:52.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:51 smithi088 bash[24895]: cephadm 2024-09-13T00:27:50.029809+0000 mgr.y (mgr.14150) 295 : cephadm [INF] Detected new or changed devices on smithi088 2024-09-13T00:27:52.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:51 smithi088 bash[24895]: cephadm 2024-09-13T00:27:50.056386+0000 mgr.y (mgr.14150) 296 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 5755M 2024-09-13T00:27:52.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:51 smithi077 bash[22596]: cephadm 2024-09-13T00:27:50.029809+0000 mgr.y (mgr.14150) 295 : cephadm [INF] Detected new or changed devices on smithi088 2024-09-13T00:27:52.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:51 smithi077 bash[22596]: cephadm 2024-09-13T00:27:50.056386+0000 mgr.y (mgr.14150) 296 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 5755M 2024-09-13T00:27:52.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:51 smithi077 bash[32040]: cephadm 2024-09-13T00:27:50.029809+0000 mgr.y (mgr.14150) 295 : cephadm [INF] Detected new or changed devices on smithi088 2024-09-13T00:27:52.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:51 smithi077 bash[32040]: cephadm 2024-09-13T00:27:50.056386+0000 mgr.y (mgr.14150) 296 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 5755M 2024-09-13T00:27:53.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:52 smithi088 bash[24895]: cluster 2024-09-13T00:27:51.266621+0000 mgr.y (mgr.14150) 297 : cluster [DBG] pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:53.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:52 smithi077 bash[22596]: cluster 2024-09-13T00:27:51.266621+0000 mgr.y (mgr.14150) 297 : cluster [DBG] pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:53.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:52 smithi077 bash[32040]: cluster 2024-09-13T00:27:51.266621+0000 mgr.y (mgr.14150) 297 : cluster [DBG] pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:54.289 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T00:27:55.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:54 smithi088 bash[24895]: cluster 2024-09-13T00:27:53.267158+0000 mgr.y (mgr.14150) 298 : cluster [DBG] pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:55.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:54 smithi077 bash[22596]: cluster 2024-09-13T00:27:53.267158+0000 mgr.y (mgr.14150) 298 : cluster [DBG] pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:55.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:54 smithi077 bash[32040]: cluster 2024-09-13T00:27:53.267158+0000 mgr.y (mgr.14150) 298 : cluster [DBG] pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:57.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:56 smithi088 bash[24895]: cluster 2024-09-13T00:27:55.267695+0000 mgr.y (mgr.14150) 299 : cluster [DBG] pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:57.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:56 smithi077 bash[22596]: cluster 2024-09-13T00:27:55.267695+0000 mgr.y (mgr.14150) 299 : cluster [DBG] pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:57.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:56 smithi077 bash[32040]: cluster 2024-09-13T00:27:55.267695+0000 mgr.y (mgr.14150) 299 : cluster [DBG] pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:58.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:57 smithi088 bash[24895]: audit 2024-09-13T00:27:57.135418+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:27:58.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:57 smithi088 bash[24895]: audit 2024-09-13T00:27:57.141437+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:27:58.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:57 smithi088 bash[24895]: audit 2024-09-13T00:27:57.143117+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:58.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:57 smithi077 bash[22596]: audit 2024-09-13T00:27:57.135418+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:27:58.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:57 smithi077 bash[22596]: audit 2024-09-13T00:27:57.141437+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:27:58.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:57 smithi077 bash[22596]: audit 2024-09-13T00:27:57.143117+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:58.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:57 smithi077 bash[32040]: audit 2024-09-13T00:27:57.135418+0000 mon.a (mon.0) 560 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-13T00:27:58.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:57 smithi077 bash[32040]: audit 2024-09-13T00:27:57.141437+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-13T00:27:58.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:57 smithi077 bash[32040]: audit 2024-09-13T00:27:57.143117+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:27:59.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:58 smithi088 bash[24895]: audit 2024-09-13T00:27:57.129140+0000 mgr.y (mgr.14150) 300 : audit [DBG] from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:27:59.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:27:58 smithi088 bash[24895]: cluster 2024-09-13T00:27:57.268225+0000 mgr.y (mgr.14150) 301 : cluster [DBG] pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:59.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:58 smithi077 bash[22596]: audit 2024-09-13T00:27:57.129140+0000 mgr.y (mgr.14150) 300 : audit [DBG] from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:27:59.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:27:58 smithi077 bash[22596]: cluster 2024-09-13T00:27:57.268225+0000 mgr.y (mgr.14150) 301 : cluster [DBG] pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:27:59.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:58 smithi077 bash[32040]: audit 2024-09-13T00:27:57.129140+0000 mgr.y (mgr.14150) 300 : audit [DBG] from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi088:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:27:59.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:27:58 smithi077 bash[32040]: cluster 2024-09-13T00:27:57.268225+0000 mgr.y (mgr.14150) 301 : cluster [DBG] pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:01.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:00 smithi077 bash[22596]: cluster 2024-09-13T00:27:59.268854+0000 mgr.y (mgr.14150) 302 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:01.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:00 smithi077 bash[32040]: cluster 2024-09-13T00:27:59.268854+0000 mgr.y (mgr.14150) 302 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:00 smithi088 bash[24895]: cluster 2024-09-13T00:27:59.268854+0000 mgr.y (mgr.14150) 302 : cluster [DBG] pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:03.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:02 smithi088 bash[24895]: cluster 2024-09-13T00:28:01.269427+0000 mgr.y (mgr.14150) 303 : cluster [DBG] pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:03.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:02 smithi077 bash[22596]: cluster 2024-09-13T00:28:01.269427+0000 mgr.y (mgr.14150) 303 : cluster [DBG] pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:03.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:02 smithi077 bash[32040]: cluster 2024-09-13T00:28:01.269427+0000 mgr.y (mgr.14150) 303 : cluster [DBG] pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:05.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:04 smithi077 bash[22596]: cluster 2024-09-13T00:28:03.270016+0000 mgr.y (mgr.14150) 304 : cluster [DBG] pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:05.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:04 smithi077 bash[32040]: cluster 2024-09-13T00:28:03.270016+0000 mgr.y (mgr.14150) 304 : cluster [DBG] pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:05.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:04 smithi088 bash[24895]: cluster 2024-09-13T00:28:03.270016+0000 mgr.y (mgr.14150) 304 : cluster [DBG] pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:06.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:05 smithi077 bash[22596]: audit 2024-09-13T00:28:05.095242+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.88:0/274726661' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b2c1a7c-3203-4e9a-b07b-4eaae7e09c0c"}]: dispatch 2024-09-13T00:28:06.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:05 smithi077 bash[22596]: audit 2024-09-13T00:28:05.096549+0000 mon.a (mon.0) 563 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b2c1a7c-3203-4e9a-b07b-4eaae7e09c0c"}]: dispatch 2024-09-13T00:28:06.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:05 smithi077 bash[22596]: audit 2024-09-13T00:28:05.105520+0000 mon.a (mon.0) 564 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2b2c1a7c-3203-4e9a-b07b-4eaae7e09c0c"}]': finished 2024-09-13T00:28:06.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:05 smithi077 bash[22596]: cluster 2024-09-13T00:28:05.105655+0000 mon.a (mon.0) 565 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2024-09-13T00:28:06.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:05 smithi077 bash[22596]: audit 2024-09-13T00:28:05.105923+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:28:06.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:05 smithi077 bash[22596]: audit 2024-09-13T00:28:05.725593+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.88:0/749653020' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:28:06.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:05 smithi077 bash[32040]: audit 2024-09-13T00:28:05.095242+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.88:0/274726661' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b2c1a7c-3203-4e9a-b07b-4eaae7e09c0c"}]: dispatch 2024-09-13T00:28:06.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:05 smithi077 bash[32040]: audit 2024-09-13T00:28:05.096549+0000 mon.a (mon.0) 563 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b2c1a7c-3203-4e9a-b07b-4eaae7e09c0c"}]: dispatch 2024-09-13T00:28:06.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:05 smithi077 bash[32040]: audit 2024-09-13T00:28:05.105520+0000 mon.a (mon.0) 564 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2b2c1a7c-3203-4e9a-b07b-4eaae7e09c0c"}]': finished 2024-09-13T00:28:06.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:05 smithi077 bash[32040]: cluster 2024-09-13T00:28:05.105655+0000 mon.a (mon.0) 565 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2024-09-13T00:28:06.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:05 smithi077 bash[32040]: audit 2024-09-13T00:28:05.105923+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:28:06.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:05 smithi077 bash[32040]: audit 2024-09-13T00:28:05.725593+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.88:0/749653020' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:28:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:05 smithi088 bash[24895]: audit 2024-09-13T00:28:05.095242+0000 mon.b (mon.1) 21 : audit [INF] from='client.? 172.21.15.88:0/274726661' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b2c1a7c-3203-4e9a-b07b-4eaae7e09c0c"}]: dispatch 2024-09-13T00:28:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:05 smithi088 bash[24895]: audit 2024-09-13T00:28:05.096549+0000 mon.a (mon.0) 563 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2b2c1a7c-3203-4e9a-b07b-4eaae7e09c0c"}]: dispatch 2024-09-13T00:28:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:05 smithi088 bash[24895]: audit 2024-09-13T00:28:05.105520+0000 mon.a (mon.0) 564 : audit [INF] from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2b2c1a7c-3203-4e9a-b07b-4eaae7e09c0c"}]': finished 2024-09-13T00:28:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:05 smithi088 bash[24895]: cluster 2024-09-13T00:28:05.105655+0000 mon.a (mon.0) 565 : cluster [DBG] osdmap e45: 8 total, 7 up, 8 in 2024-09-13T00:28:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:05 smithi088 bash[24895]: audit 2024-09-13T00:28:05.105923+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:28:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:05 smithi088 bash[24895]: audit 2024-09-13T00:28:05.725593+0000 mon.b (mon.1) 22 : audit [DBG] from='client.? 172.21.15.88:0/749653020' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-13T00:28:07.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:06 smithi077 bash[22596]: cluster 2024-09-13T00:28:05.270612+0000 mgr.y (mgr.14150) 305 : cluster [DBG] pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:07.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:06 smithi077 bash[32040]: cluster 2024-09-13T00:28:05.270612+0000 mgr.y (mgr.14150) 305 : cluster [DBG] pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:07.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:06 smithi088 bash[24895]: cluster 2024-09-13T00:28:05.270612+0000 mgr.y (mgr.14150) 305 : cluster [DBG] pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:09.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:08 smithi077 bash[32040]: cluster 2024-09-13T00:28:07.271131+0000 mgr.y (mgr.14150) 306 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:09.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:08 smithi077 bash[22596]: cluster 2024-09-13T00:28:07.271131+0000 mgr.y (mgr.14150) 306 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:09.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:08 smithi088 bash[24895]: cluster 2024-09-13T00:28:07.271131+0000 mgr.y (mgr.14150) 306 : cluster [DBG] pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:11.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:10 smithi077 bash[32040]: cluster 2024-09-13T00:28:09.271657+0000 mgr.y (mgr.14150) 307 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:11.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:10 smithi077 bash[22596]: cluster 2024-09-13T00:28:09.271657+0000 mgr.y (mgr.14150) 307 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:11.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:10 smithi088 bash[24895]: cluster 2024-09-13T00:28:09.271657+0000 mgr.y (mgr.14150) 307 : cluster [DBG] pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:13.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:12 smithi077 bash[32040]: cluster 2024-09-13T00:28:11.272232+0000 mgr.y (mgr.14150) 308 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:13.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:12 smithi077 bash[22596]: cluster 2024-09-13T00:28:11.272232+0000 mgr.y (mgr.14150) 308 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:13.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:12 smithi088 bash[24895]: cluster 2024-09-13T00:28:11.272232+0000 mgr.y (mgr.14150) 308 : cluster [DBG] pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:15.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:14 smithi088 bash[24895]: cluster 2024-09-13T00:28:13.272778+0000 mgr.y (mgr.14150) 309 : cluster [DBG] pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:15.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:14 smithi077 bash[22596]: cluster 2024-09-13T00:28:13.272778+0000 mgr.y (mgr.14150) 309 : cluster [DBG] pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:15.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:14 smithi077 bash[32040]: cluster 2024-09-13T00:28:13.272778+0000 mgr.y (mgr.14150) 309 : cluster [DBG] pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:17.227 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:16 smithi088 bash[24895]: cluster 2024-09-13T00:28:15.273343+0000 mgr.y (mgr.14150) 310 : cluster [DBG] pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:17.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:16 smithi077 bash[22596]: cluster 2024-09-13T00:28:15.273343+0000 mgr.y (mgr.14150) 310 : cluster [DBG] pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:17.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:16 smithi077 bash[32040]: cluster 2024-09-13T00:28:15.273343+0000 mgr.y (mgr.14150) 310 : cluster [DBG] pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:18.163 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:17 smithi088 bash[24895]: audit 2024-09-13T00:28:17.011569+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T00:28:18.164 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:17 smithi088 bash[24895]: audit 2024-09-13T00:28:17.013087+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:28:18.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:17 smithi077 bash[22596]: audit 2024-09-13T00:28:17.011569+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T00:28:18.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:17 smithi077 bash[22596]: audit 2024-09-13T00:28:17.013087+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:28:18.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:17 smithi077 bash[32040]: audit 2024-09-13T00:28:17.011569+0000 mon.a (mon.0) 567 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T00:28:18.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:17 smithi077 bash[32040]: audit 2024-09-13T00:28:17.013087+0000 mon.a (mon.0) 568 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:28:18.896 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:18 smithi077 bash[22596]: cephadm 2024-09-13T00:28:17.014434+0000 mgr.y (mgr.14150) 311 : cephadm [INF] Deploying daemon osd.7 on smithi088 2024-09-13T00:28:18.897 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:18 smithi077 bash[22596]: cluster 2024-09-13T00:28:17.273895+0000 mgr.y (mgr.14150) 312 : cluster [DBG] pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:19.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:18 smithi077 bash[32040]: cephadm 2024-09-13T00:28:17.014434+0000 mgr.y (mgr.14150) 311 : cephadm [INF] Deploying daemon osd.7 on smithi088 2024-09-13T00:28:19.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:18 smithi077 bash[32040]: cluster 2024-09-13T00:28:17.273895+0000 mgr.y (mgr.14150) 312 : cluster [DBG] pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:19.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:18 smithi088 bash[24895]: cephadm 2024-09-13T00:28:17.014434+0000 mgr.y (mgr.14150) 311 : cephadm [INF] Deploying daemon osd.7 on smithi088 2024-09-13T00:28:19.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:18 smithi088 bash[24895]: cluster 2024-09-13T00:28:17.273895+0000 mgr.y (mgr.14150) 312 : cluster [DBG] pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:20.832 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:28:20 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:28:20.833 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:20 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:28:20.833 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:28:20 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:28:20.833 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:28:20 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:28:20.834 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:28:20 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:28:21.101 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:28:20 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:28:21.101 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:28:20 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:28:21.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:20 smithi088 bash[24895]: cluster 2024-09-13T00:28:19.274426+0000 mgr.y (mgr.14150) 313 : cluster [DBG] pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:21.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:20 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:28:21.102 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:28:20 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:28:21.102 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:28:20 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:28:21.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:20 smithi077 bash[22596]: cluster 2024-09-13T00:28:19.274426+0000 mgr.y (mgr.14150) 313 : cluster [DBG] pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:21.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:20 smithi077 bash[32040]: cluster 2024-09-13T00:28:19.274426+0000 mgr.y (mgr.14150) 313 : cluster [DBG] pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:22.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:21 smithi077 bash[32040]: audit 2024-09-13T00:28:21.122983+0000 mon.a (mon.0) 569 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:28:22.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:21 smithi077 bash[32040]: audit 2024-09-13T00:28:21.130515+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:22.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:21 smithi077 bash[32040]: audit 2024-09-13T00:28:21.139140+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:22.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:21 smithi077 bash[22596]: audit 2024-09-13T00:28:21.122983+0000 mon.a (mon.0) 569 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:28:22.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:21 smithi077 bash[22596]: audit 2024-09-13T00:28:21.130515+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:22.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:21 smithi077 bash[22596]: audit 2024-09-13T00:28:21.139140+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:22.348 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:21 smithi088 bash[24895]: audit 2024-09-13T00:28:21.122983+0000 mon.a (mon.0) 569 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:28:22.348 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:21 smithi088 bash[24895]: audit 2024-09-13T00:28:21.130515+0000 mon.a (mon.0) 570 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:22.348 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:21 smithi088 bash[24895]: audit 2024-09-13T00:28:21.139140+0000 mon.a (mon.0) 571 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:23.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:22 smithi077 bash[32040]: cluster 2024-09-13T00:28:21.274975+0000 mgr.y (mgr.14150) 314 : cluster [DBG] pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:23.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:22 smithi077 bash[22596]: cluster 2024-09-13T00:28:21.274975+0000 mgr.y (mgr.14150) 314 : cluster [DBG] pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:23.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:22 smithi088 bash[24895]: cluster 2024-09-13T00:28:21.274975+0000 mgr.y (mgr.14150) 314 : cluster [DBG] pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:25.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:24 smithi077 bash[32040]: cluster 2024-09-13T00:28:23.275508+0000 mgr.y (mgr.14150) 315 : cluster [DBG] pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:25.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:24 smithi077 bash[22596]: cluster 2024-09-13T00:28:23.275508+0000 mgr.y (mgr.14150) 315 : cluster [DBG] pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:25.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:24 smithi088 bash[24895]: cluster 2024-09-13T00:28:23.275508+0000 mgr.y (mgr.14150) 315 : cluster [DBG] pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:27.213 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:26 smithi088 bash[24895]: cluster 2024-09-13T00:28:25.276035+0000 mgr.y (mgr.14150) 316 : cluster [DBG] pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:27.214 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:26 smithi088 bash[24895]: audit 2024-09-13T00:28:26.267367+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:27.214 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:26 smithi088 bash[24895]: audit 2024-09-13T00:28:26.278422+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:27.214 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:26 smithi088 bash[24895]: audit 2024-09-13T00:28:26.280293+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:28:27.214 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:26 smithi088 bash[24895]: audit 2024-09-13T00:28:26.282148+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:28:27.214 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:26 smithi088 bash[24895]: audit 2024-09-13T00:28:26.294338+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:27.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:26 smithi077 bash[22596]: cluster 2024-09-13T00:28:25.276035+0000 mgr.y (mgr.14150) 316 : cluster [DBG] pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:27.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:26 smithi077 bash[22596]: audit 2024-09-13T00:28:26.267367+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:27.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:26 smithi077 bash[22596]: audit 2024-09-13T00:28:26.278422+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:27.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:26 smithi077 bash[22596]: audit 2024-09-13T00:28:26.280293+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:28:27.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:26 smithi077 bash[22596]: audit 2024-09-13T00:28:26.282148+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:28:27.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:26 smithi077 bash[22596]: audit 2024-09-13T00:28:26.294338+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:27.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:26 smithi077 bash[32040]: cluster 2024-09-13T00:28:25.276035+0000 mgr.y (mgr.14150) 316 : cluster [DBG] pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:27.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:26 smithi077 bash[32040]: audit 2024-09-13T00:28:26.267367+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:27.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:26 smithi077 bash[32040]: audit 2024-09-13T00:28:26.278422+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:27.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:26 smithi077 bash[32040]: audit 2024-09-13T00:28:26.280293+0000 mon.a (mon.0) 574 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:28:27.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:26 smithi077 bash[32040]: audit 2024-09-13T00:28:26.282148+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:28:27.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:26 smithi077 bash[32040]: audit 2024-09-13T00:28:26.294338+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:28.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:27 smithi077 bash[22596]: audit 2024-09-13T00:28:27.389218+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.88:6824/2009347492,v1:172.21.15.88:6825/2009347492]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T00:28:28.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:27 smithi077 bash[22596]: audit 2024-09-13T00:28:27.390273+0000 mon.a (mon.0) 577 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T00:28:28.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:27 smithi077 bash[32040]: audit 2024-09-13T00:28:27.389218+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.88:6824/2009347492,v1:172.21.15.88:6825/2009347492]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T00:28:28.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:27 smithi077 bash[32040]: audit 2024-09-13T00:28:27.390273+0000 mon.a (mon.0) 577 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T00:28:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:27 smithi088 bash[24895]: audit 2024-09-13T00:28:27.389218+0000 mon.b (mon.1) 23 : audit [INF] from='osd.7 [v2:172.21.15.88:6824/2009347492,v1:172.21.15.88:6825/2009347492]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T00:28:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:27 smithi088 bash[24895]: audit 2024-09-13T00:28:27.390273+0000 mon.a (mon.0) 577 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-13T00:28:29.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:28 smithi077 bash[32040]: cluster 2024-09-13T00:28:27.276594+0000 mgr.y (mgr.14150) 317 : cluster [DBG] pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:29.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:28 smithi077 bash[32040]: audit 2024-09-13T00:28:27.931447+0000 mon.a (mon.0) 578 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T00:28:29.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:28 smithi077 bash[32040]: cluster 2024-09-13T00:28:27.931654+0000 mon.a (mon.0) 579 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2024-09-13T00:28:29.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:28 smithi077 bash[32040]: audit 2024-09-13T00:28:27.932656+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.88:6824/2009347492,v1:172.21.15.88:6825/2009347492]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:28:29.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:28 smithi077 bash[32040]: audit 2024-09-13T00:28:27.932994+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:28:29.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:28 smithi077 bash[32040]: audit 2024-09-13T00:28:27.933741+0000 mon.a (mon.0) 581 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:28:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:28 smithi077 bash[22596]: cluster 2024-09-13T00:28:27.276594+0000 mgr.y (mgr.14150) 317 : cluster [DBG] pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:28 smithi077 bash[22596]: audit 2024-09-13T00:28:27.931447+0000 mon.a (mon.0) 578 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T00:28:29.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:28 smithi077 bash[22596]: cluster 2024-09-13T00:28:27.931654+0000 mon.a (mon.0) 579 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2024-09-13T00:28:29.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:28 smithi077 bash[22596]: audit 2024-09-13T00:28:27.932656+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.88:6824/2009347492,v1:172.21.15.88:6825/2009347492]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:28:29.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:28 smithi077 bash[22596]: audit 2024-09-13T00:28:27.932994+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:28:29.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:28 smithi077 bash[22596]: audit 2024-09-13T00:28:27.933741+0000 mon.a (mon.0) 581 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:28:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:28 smithi088 bash[24895]: cluster 2024-09-13T00:28:27.276594+0000 mgr.y (mgr.14150) 317 : cluster [DBG] pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:28 smithi088 bash[24895]: audit 2024-09-13T00:28:27.931447+0000 mon.a (mon.0) 578 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-13T00:28:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:28 smithi088 bash[24895]: cluster 2024-09-13T00:28:27.931654+0000 mon.a (mon.0) 579 : cluster [DBG] osdmap e46: 8 total, 7 up, 8 in 2024-09-13T00:28:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:28 smithi088 bash[24895]: audit 2024-09-13T00:28:27.932656+0000 mon.b (mon.1) 24 : audit [INF] from='osd.7 [v2:172.21.15.88:6824/2009347492,v1:172.21.15.88:6825/2009347492]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:28:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:28 smithi088 bash[24895]: audit 2024-09-13T00:28:27.932994+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:28:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:28 smithi088 bash[24895]: audit 2024-09-13T00:28:27.933741+0000 mon.a (mon.0) 581 : audit [INF] from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:28:29.607 INFO:teuthology.orchestra.run.smithi088.stdout:Created osd(s) 7 on host 'smithi088' 2024-09-13T00:28:30.190 DEBUG:teuthology.orchestra.run.smithi088:osd.7> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.7.service 2024-09-13T00:28:30.193 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-13T00:28:30.193 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-13T00:28:30.201 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:29 smithi088 bash[24895]: audit 2024-09-13T00:28:28.933919+0000 mon.a (mon.0) 582 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-13T00:28:30.201 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:29 smithi088 bash[24895]: cluster 2024-09-13T00:28:28.934273+0000 mon.a (mon.0) 583 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2024-09-13T00:28:30.201 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:29 smithi088 bash[24895]: audit 2024-09-13T00:28:28.935143+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:28:30.201 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:29 smithi088 bash[24895]: audit 2024-09-13T00:28:28.944370+0000 mon.a (mon.0) 585 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:28:30.202 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:29 smithi088 bash[24895]: cluster 2024-09-13T00:28:29.277149+0000 mgr.y (mgr.14150) 318 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:30.202 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:29 smithi088 bash[24895]: audit 2024-09-13T00:28:29.589628+0000 mon.a (mon.0) 586 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:28:30.202 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:29 smithi088 bash[24895]: audit 2024-09-13T00:28:29.597643+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:30.202 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:29 smithi088 bash[24895]: audit 2024-09-13T00:28:29.606425+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:30.202 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:29 smithi088 bash[24895]: cluster 2024-09-13T00:28:29.935794+0000 mon.a (mon.0) 589 : cluster [INF] osd.7 [v2:172.21.15.88:6824/2009347492,v1:172.21.15.88:6825/2009347492] boot 2024-09-13T00:28:30.202 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:29 smithi088 bash[24895]: cluster 2024-09-13T00:28:29.935899+0000 mon.a (mon.0) 590 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2024-09-13T00:28:30.204 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[22596]: audit 2024-09-13T00:28:28.933919+0000 mon.a (mon.0) 582 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-13T00:28:30.204 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[22596]: cluster 2024-09-13T00:28:28.934273+0000 mon.a (mon.0) 583 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2024-09-13T00:28:30.205 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[22596]: audit 2024-09-13T00:28:28.935143+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:28:30.205 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[22596]: audit 2024-09-13T00:28:28.944370+0000 mon.a (mon.0) 585 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:28:30.205 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[22596]: cluster 2024-09-13T00:28:29.277149+0000 mgr.y (mgr.14150) 318 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:30.205 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[22596]: audit 2024-09-13T00:28:29.589628+0000 mon.a (mon.0) 586 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:28:30.205 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[22596]: audit 2024-09-13T00:28:29.597643+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:30.205 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[22596]: audit 2024-09-13T00:28:29.606425+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:30.205 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[22596]: cluster 2024-09-13T00:28:29.935794+0000 mon.a (mon.0) 589 : cluster [INF] osd.7 [v2:172.21.15.88:6824/2009347492,v1:172.21.15.88:6825/2009347492] boot 2024-09-13T00:28:30.205 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[22596]: cluster 2024-09-13T00:28:29.935899+0000 mon.a (mon.0) 590 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2024-09-13T00:28:30.205 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[32040]: audit 2024-09-13T00:28:28.933919+0000 mon.a (mon.0) 582 : audit [INF] from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]': finished 2024-09-13T00:28:30.206 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[32040]: cluster 2024-09-13T00:28:28.934273+0000 mon.a (mon.0) 583 : cluster [DBG] osdmap e47: 8 total, 7 up, 8 in 2024-09-13T00:28:30.206 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[32040]: audit 2024-09-13T00:28:28.935143+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:28:30.206 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[32040]: audit 2024-09-13T00:28:28.944370+0000 mon.a (mon.0) 585 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:28:30.206 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[32040]: cluster 2024-09-13T00:28:29.277149+0000 mgr.y (mgr.14150) 318 : cluster [DBG] pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-09-13T00:28:30.206 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[32040]: audit 2024-09-13T00:28:29.589628+0000 mon.a (mon.0) 586 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:28:30.206 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[32040]: audit 2024-09-13T00:28:29.597643+0000 mon.a (mon.0) 587 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:30.206 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[32040]: audit 2024-09-13T00:28:29.606425+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:30.206 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[32040]: cluster 2024-09-13T00:28:29.935794+0000 mon.a (mon.0) 589 : cluster [INF] osd.7 [v2:172.21.15.88:6824/2009347492,v1:172.21.15.88:6825/2009347492] boot 2024-09-13T00:28:30.206 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:29 smithi077 bash[32040]: cluster 2024-09-13T00:28:29.935899+0000 mon.a (mon.0) 590 : cluster [DBG] osdmap e48: 8 total, 8 up, 8 in 2024-09-13T00:28:31.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:30 smithi077 bash[22596]: cluster 2024-09-13T00:28:28.382950+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:28:31.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:30 smithi077 bash[22596]: cluster 2024-09-13T00:28:28.383049+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:28:31.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:30 smithi077 bash[22596]: audit 2024-09-13T00:28:29.937198+0000 mon.a (mon.0) 591 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:28:31.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:30 smithi077 bash[22596]: cluster 2024-09-13T00:28:30.814440+0000 mon.a (mon.0) 592 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2024-09-13T00:28:31.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:30 smithi077 bash[32040]: cluster 2024-09-13T00:28:28.382950+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:28:31.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:30 smithi077 bash[32040]: cluster 2024-09-13T00:28:28.383049+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:28:31.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:30 smithi077 bash[32040]: audit 2024-09-13T00:28:29.937198+0000 mon.a (mon.0) 591 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:28:31.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:30 smithi077 bash[32040]: cluster 2024-09-13T00:28:30.814440+0000 mon.a (mon.0) 592 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2024-09-13T00:28:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:30 smithi088 bash[24895]: cluster 2024-09-13T00:28:28.382950+0000 osd.7 (osd.7) 1 : cluster [DBG] purged_snaps scrub starts 2024-09-13T00:28:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:30 smithi088 bash[24895]: cluster 2024-09-13T00:28:28.383049+0000 osd.7 (osd.7) 2 : cluster [DBG] purged_snaps scrub ok 2024-09-13T00:28:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:30 smithi088 bash[24895]: audit 2024-09-13T00:28:29.937198+0000 mon.a (mon.0) 591 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:28:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:30 smithi088 bash[24895]: cluster 2024-09-13T00:28:30.814440+0000 mon.a (mon.0) 592 : cluster [DBG] osdmap e49: 8 total, 8 up, 8 in 2024-09-13T00:28:33.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:32 smithi088 bash[24895]: cluster 2024-09-13T00:28:31.277759+0000 mgr.y (mgr.14150) 319 : cluster [DBG] pgmap v297: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:33.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:32 smithi088 bash[24895]: cluster 2024-09-13T00:28:31.815805+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2024-09-13T00:28:33.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:32 smithi088 bash[24895]: cluster 2024-09-13T00:28:31.946523+0000 mon.a (mon.0) 594 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T00:28:33.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:32 smithi077 bash[32040]: cluster 2024-09-13T00:28:31.277759+0000 mgr.y (mgr.14150) 319 : cluster [DBG] pgmap v297: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:33.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:32 smithi077 bash[32040]: cluster 2024-09-13T00:28:31.815805+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2024-09-13T00:28:33.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:32 smithi077 bash[32040]: cluster 2024-09-13T00:28:31.946523+0000 mon.a (mon.0) 594 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T00:28:33.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:32 smithi077 bash[22596]: cluster 2024-09-13T00:28:31.277759+0000 mgr.y (mgr.14150) 319 : cluster [DBG] pgmap v297: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:33.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:32 smithi077 bash[22596]: cluster 2024-09-13T00:28:31.815805+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e50: 8 total, 8 up, 8 in 2024-09-13T00:28:33.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:32 smithi077 bash[22596]: cluster 2024-09-13T00:28:31.946523+0000 mon.a (mon.0) 594 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-13T00:28:34.948 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:28:35.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:34 smithi088 bash[24895]: cluster 2024-09-13T00:28:33.278368+0000 mgr.y (mgr.14150) 320 : cluster [DBG] pgmap v299: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:35.107 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:34 smithi077 bash[32040]: cluster 2024-09-13T00:28:33.278368+0000 mgr.y (mgr.14150) 320 : cluster [DBG] pgmap v299: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:35.107 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:34 smithi077 bash[22596]: cluster 2024-09-13T00:28:33.278368+0000 mgr.y (mgr.14150) 320 : cluster [DBG] pgmap v299: 1 pgs: 1 peering; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:36.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:35 smithi088 bash[24895]: cluster 2024-09-13T00:28:35.817093+0000 mon.a (mon.0) 595 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T00:28:36.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:35 smithi088 bash[24895]: cluster 2024-09-13T00:28:35.817148+0000 mon.a (mon.0) 596 : cluster [INF] Cluster is now healthy 2024-09-13T00:28:36.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:35 smithi077 bash[32040]: cluster 2024-09-13T00:28:35.817093+0000 mon.a (mon.0) 595 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T00:28:36.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:35 smithi077 bash[32040]: cluster 2024-09-13T00:28:35.817148+0000 mon.a (mon.0) 596 : cluster [INF] Cluster is now healthy 2024-09-13T00:28:36.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:35 smithi077 bash[22596]: cluster 2024-09-13T00:28:35.817093+0000 mon.a (mon.0) 595 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-13T00:28:36.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:35 smithi077 bash[22596]: cluster 2024-09-13T00:28:35.817148+0000 mon.a (mon.0) 596 : cluster [INF] Cluster is now healthy 2024-09-13T00:28:37.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:36 smithi077 bash[22596]: cluster 2024-09-13T00:28:35.278931+0000 mgr.y (mgr.14150) 321 : cluster [DBG] pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:37.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:36 smithi077 bash[32040]: cluster 2024-09-13T00:28:35.278931+0000 mgr.y (mgr.14150) 321 : cluster [DBG] pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:37.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:36 smithi088 bash[24895]: cluster 2024-09-13T00:28:35.278931+0000 mgr.y (mgr.14150) 321 : cluster [DBG] pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:38.890 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:28:39.124 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[32040]: cluster 2024-09-13T00:28:37.279336+0000 mgr.y (mgr.14150) 322 : cluster [DBG] pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-09-13T00:28:39.125 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[32040]: audit 2024-09-13T00:28:38.606219+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:39.125 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[32040]: audit 2024-09-13T00:28:38.611914+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:39.125 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[32040]: audit 2024-09-13T00:28:38.613154+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:28:39.125 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[32040]: audit 2024-09-13T00:28:38.614029+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:28:39.125 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[32040]: audit 2024-09-13T00:28:38.614712+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:28:39.125 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[32040]: audit 2024-09-13T00:28:38.615367+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:28:39.125 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[32040]: audit 2024-09-13T00:28:38.620541+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:39.125 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[32040]: audit 2024-09-13T00:28:38.621430+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:28:39.125 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[32040]: audit 2024-09-13T00:28:38.622240+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:28:39.125 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[32040]: audit 2024-09-13T00:28:38.627109+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:39.126 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[22596]: cluster 2024-09-13T00:28:37.279336+0000 mgr.y (mgr.14150) 322 : cluster [DBG] pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-09-13T00:28:39.126 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[22596]: audit 2024-09-13T00:28:38.606219+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:39.126 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[22596]: audit 2024-09-13T00:28:38.611914+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:39.126 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[22596]: audit 2024-09-13T00:28:38.613154+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:28:39.126 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[22596]: audit 2024-09-13T00:28:38.614029+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:28:39.126 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[22596]: audit 2024-09-13T00:28:38.614712+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:28:39.126 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[22596]: audit 2024-09-13T00:28:38.615367+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:28:39.126 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[22596]: audit 2024-09-13T00:28:38.620541+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:39.126 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[22596]: audit 2024-09-13T00:28:38.621430+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:28:39.127 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[22596]: audit 2024-09-13T00:28:38.622240+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:28:39.127 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:38 smithi077 bash[22596]: audit 2024-09-13T00:28:38.627109+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:39.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:38 smithi088 bash[24895]: cluster 2024-09-13T00:28:37.279336+0000 mgr.y (mgr.14150) 322 : cluster [DBG] pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-09-13T00:28:39.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:38 smithi088 bash[24895]: audit 2024-09-13T00:28:38.606219+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:39.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:38 smithi088 bash[24895]: audit 2024-09-13T00:28:38.611914+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:39.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:38 smithi088 bash[24895]: audit 2024-09-13T00:28:38.613154+0000 mon.a (mon.0) 599 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:28:39.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:38 smithi088 bash[24895]: audit 2024-09-13T00:28:38.614029+0000 mon.a (mon.0) 600 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:28:39.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:38 smithi088 bash[24895]: audit 2024-09-13T00:28:38.614712+0000 mon.a (mon.0) 601 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:28:39.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:38 smithi088 bash[24895]: audit 2024-09-13T00:28:38.615367+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:28:39.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:38 smithi088 bash[24895]: audit 2024-09-13T00:28:38.620541+0000 mon.a (mon.0) 603 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:39.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:38 smithi088 bash[24895]: audit 2024-09-13T00:28:38.621430+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:28:39.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:38 smithi088 bash[24895]: audit 2024-09-13T00:28:38.622240+0000 mon.a (mon.0) 605 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:28:39.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:38 smithi088 bash[24895]: audit 2024-09-13T00:28:38.627109+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:28:39.702 INFO:teuthology.orchestra.run.smithi077.stdout:{"epoch":50,"num_osds":8,"num_up_osds":8,"osd_up_since":1726187309,"num_in_osds":8,"osd_in_since":1726187285,"num_remapped_pgs":0} 2024-09-13T00:28:39.702 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T00:28:40.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:39 smithi077 bash[22596]: cephadm 2024-09-13T00:28:38.600772+0000 mgr.y (mgr.14150) 323 : cephadm [INF] Detected new or changed devices on smithi088 2024-09-13T00:28:40.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:39 smithi077 bash[22596]: cephadm 2024-09-13T00:28:38.615869+0000 mgr.y (mgr.14150) 324 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 4316M 2024-09-13T00:28:40.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:39 smithi077 bash[22596]: audit 2024-09-13T00:28:38.893880+0000 mon.a (mon.0) 607 : audit [DBG] from='client.? 172.21.15.77:0/1855454856' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T00:28:40.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:39 smithi077 bash[32040]: cephadm 2024-09-13T00:28:38.600772+0000 mgr.y (mgr.14150) 323 : cephadm [INF] Detected new or changed devices on smithi088 2024-09-13T00:28:40.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:39 smithi077 bash[32040]: cephadm 2024-09-13T00:28:38.615869+0000 mgr.y (mgr.14150) 324 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 4316M 2024-09-13T00:28:40.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:39 smithi077 bash[32040]: audit 2024-09-13T00:28:38.893880+0000 mon.a (mon.0) 607 : audit [DBG] from='client.? 172.21.15.77:0/1855454856' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T00:28:40.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:39 smithi088 bash[24895]: cephadm 2024-09-13T00:28:38.600772+0000 mgr.y (mgr.14150) 323 : cephadm [INF] Detected new or changed devices on smithi088 2024-09-13T00:28:40.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:39 smithi088 bash[24895]: cephadm 2024-09-13T00:28:38.615869+0000 mgr.y (mgr.14150) 324 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 4316M 2024-09-13T00:28:40.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:39 smithi088 bash[24895]: audit 2024-09-13T00:28:38.893880+0000 mon.a (mon.0) 607 : audit [DBG] from='client.? 172.21.15.77:0/1855454856' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-13T00:28:41.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:40 smithi077 bash[22596]: cluster 2024-09-13T00:28:39.279968+0000 mgr.y (mgr.14150) 325 : cluster [DBG] pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-09-13T00:28:41.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:40 smithi077 bash[32040]: cluster 2024-09-13T00:28:39.279968+0000 mgr.y (mgr.14150) 325 : cluster [DBG] pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-09-13T00:28:41.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:40 smithi088 bash[24895]: cluster 2024-09-13T00:28:39.279968+0000 mgr.y (mgr.14150) 325 : cluster [DBG] pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s, 0 objects/s recovering 2024-09-13T00:28:43.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:42 smithi077 bash[32040]: cluster 2024-09-13T00:28:41.280707+0000 mgr.y (mgr.14150) 326 : cluster [DBG] pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T00:28:43.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:42 smithi077 bash[22596]: cluster 2024-09-13T00:28:41.280707+0000 mgr.y (mgr.14150) 326 : cluster [DBG] pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T00:28:43.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:42 smithi088 bash[24895]: cluster 2024-09-13T00:28:41.280707+0000 mgr.y (mgr.14150) 326 : cluster [DBG] pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-13T00:28:44.505 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:28:45.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:44 smithi077 bash[22596]: cluster 2024-09-13T00:28:43.281394+0000 mgr.y (mgr.14150) 327 : cluster [DBG] pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-09-13T00:28:45.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:44 smithi077 bash[32040]: cluster 2024-09-13T00:28:43.281394+0000 mgr.y (mgr.14150) 327 : cluster [DBG] pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-09-13T00:28:45.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:44 smithi088 bash[24895]: cluster 2024-09-13T00:28:43.281394+0000 mgr.y (mgr.14150) 327 : cluster [DBG] pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s, 0 objects/s recovering 2024-09-13T00:28:47.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:46 smithi077 bash[22596]: cluster 2024-09-13T00:28:45.282009+0000 mgr.y (mgr.14150) 328 : cluster [DBG] pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T00:28:47.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:46 smithi077 bash[32040]: cluster 2024-09-13T00:28:45.282009+0000 mgr.y (mgr.14150) 328 : cluster [DBG] pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T00:28:47.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:46 smithi088 bash[24895]: cluster 2024-09-13T00:28:45.282009+0000 mgr.y (mgr.14150) 328 : cluster [DBG] pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-13T00:28:48.388 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:28:48.388 INFO:teuthology.orchestra.run.smithi077.stdout:{"epoch":50,"fsid":"9218306c-7165-11ef-bceb-c7b262605968","created":"2024-09-13T00:18:59.375829+0000","modified":"2024-09-13T00:28:31.808080+0000","last_up_change":"2024-09-13T00:28:29.926728+0000","last_in_change":"2024-09-13T00:28:05.097746+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T00:24:15.726444+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"e0860feb-72c9-4c49-933d-0a49b68caeee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6802","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6803","nonce":1623816886}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6804","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6805","nonce":1623816886}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6808","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6809","nonce":1623816886}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6806","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6807","nonce":1623816886}]},"public_addr":"172.21.15.77:6803/1623816886","cluster_addr":"172.21.15.77:6805/1623816886","heartbeat_back_addr":"172.21.15.77:6809/1623816886","heartbeat_front_addr":"172.21.15.77:6807/1623816886","state":["exists","up"]},{"osd":1,"uuid":"7ee01388-61b0-4a60-81f2-08c3c072ec0e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":31,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6810","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6811","nonce":1691613782}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6812","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6813","nonce":1691613782}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6816","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6817","nonce":1691613782}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6814","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6815","nonce":1691613782}]},"public_addr":"172.21.15.77:6811/1691613782","cluster_addr":"172.21.15.77:6813/1691613782","heartbeat_back_addr":"172.21.15.77:6817/1691613782","heartbeat_front_addr":"172.21.15.77:6815/1691613782","state":["exists","up"]},{"osd":2,"uuid":"ff8bc88a-5b11-469f-912d-d3149036b450","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6818","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6819","nonce":2379210664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6820","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6821","nonce":2379210664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6824","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6825","nonce":2379210664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6822","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6823","nonce":2379210664}]},"public_addr":"172.21.15.77:6819/2379210664","cluster_addr":"172.21.15.77:6821/2379210664","heartbeat_back_addr":"172.21.15.77:6825/2379210664","heartbeat_front_addr":"172.21.15.77:6823/2379210664","state":["exists","up"]},{"osd":3,"uuid":"cb032137-609e-41f6-bfc7-e70b77ea9456","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6826","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6827","nonce":2636296765}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6828","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6829","nonce":2636296765}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6832","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6833","nonce":2636296765}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6830","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6831","nonce":2636296765}]},"public_addr":"172.21.15.77:6827/2636296765","cluster_addr":"172.21.15.77:6829/2636296765","heartbeat_back_addr":"172.21.15.77:6833/2636296765","heartbeat_front_addr":"172.21.15.77:6831/2636296765","state":["exists","up"]},{"osd":4,"uuid":"cd38603e-5cdc-42b6-b8c8-656d44df0ed5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6800","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6801","nonce":3645723145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6802","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6803","nonce":3645723145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6806","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6807","nonce":3645723145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6804","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6805","nonce":3645723145}]},"public_addr":"172.21.15.88:6801/3645723145","cluster_addr":"172.21.15.88:6803/3645723145","heartbeat_back_addr":"172.21.15.88:6807/3645723145","heartbeat_front_addr":"172.21.15.88:6805/3645723145","state":["exists","up"]},{"osd":5,"uuid":"d4b81ff3-f257-4f63-a7a7-cbd1ff754544","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6808","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6809","nonce":2699666173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6810","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6811","nonce":2699666173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6814","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6815","nonce":2699666173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6812","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6813","nonce":2699666173}]},"public_addr":"172.21.15.88:6809/2699666173","cluster_addr":"172.21.15.88:6811/2699666173","heartbeat_back_addr":"172.21.15.88:6815/2699666173","heartbeat_front_addr":"172.21.15.88:6813/2699666173","state":["exists","up"]},{"osd":6,"uuid":"31c0d169-c329-440d-848b-057fbbf418ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":43,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6816","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6817","nonce":412460705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6818","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6819","nonce":412460705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6822","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6823","nonce":412460705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6820","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6821","nonce":412460705}]},"public_addr":"172.21.15.88:6817/412460705","cluster_addr":"172.21.15.88:6819/412460705","heartbeat_back_addr":"172.21.15.88:6823/412460705","heartbeat_front_addr":"172.21.15.88:6821/412460705","state":["exists","up"]},{"osd":7,"uuid":"2b2c1a7c-3203-4e9a-b07b-4eaae7e09c0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6824","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6825","nonce":2009347492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6826","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6827","nonce":2009347492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6830","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6831","nonce":2009347492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6828","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6829","nonce":2009347492}]},"public_addr":"172.21.15.88:6825/2009347492","cluster_addr":"172.21.15.88:6827/2009347492","heartbeat_back_addr":"172.21.15.88:6831/2009347492","heartbeat_front_addr":"172.21.15.88:6829/2009347492","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:22:21.643909+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:23:17.132405+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:24:13.726010+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:25:11.211228+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:26:00.949032+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:26:49.596019+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:27:38.694576+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:28:28.383054+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.77:6801/3213372405":"2024-09-14T00:19:55.144762+0000","172.21.15.77:6800/3213372405":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/2198980039":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/4117010497":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/1897411349":"2024-09-14T00:19:55.144762+0000","172.21.15.77:6801/426595304":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6800/426595304":"2024-09-14T00:19:25.064947+0000","172.21.15.77:0/4291417054":"2024-09-14T00:19:25.064947+0000","172.21.15.77:0/1137737372":"2024-09-14T00:19:25.064947+0000","172.21.15.77:0/892433806":"2024-09-14T00:19:25.064947+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T00:28:49.117 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-13T00:24:15.726444+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-09-13T00:28:49.117 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-13T00:28:49.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:48 smithi077 bash[22596]: cluster 2024-09-13T00:28:47.282542+0000 mgr.y (mgr.14150) 329 : cluster [DBG] pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:49.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:48 smithi077 bash[22596]: audit 2024-09-13T00:28:48.390232+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.77:0/192275367' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:28:49.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:48 smithi077 bash[32040]: cluster 2024-09-13T00:28:47.282542+0000 mgr.y (mgr.14150) 329 : cluster [DBG] pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:49.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:48 smithi077 bash[32040]: audit 2024-09-13T00:28:48.390232+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.77:0/192275367' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:28:49.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:48 smithi088 bash[24895]: cluster 2024-09-13T00:28:47.282542+0000 mgr.y (mgr.14150) 329 : cluster [DBG] pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:49.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:48 smithi088 bash[24895]: audit 2024-09-13T00:28:48.390232+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.77:0/192275367' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:28:51.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:50 smithi077 bash[22596]: cluster 2024-09-13T00:28:49.283050+0000 mgr.y (mgr.14150) 330 : cluster [DBG] pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:51.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:50 smithi077 bash[32040]: cluster 2024-09-13T00:28:49.283050+0000 mgr.y (mgr.14150) 330 : cluster [DBG] pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:51.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:50 smithi088 bash[24895]: cluster 2024-09-13T00:28:49.283050+0000 mgr.y (mgr.14150) 330 : cluster [DBG] pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:53.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:52 smithi077 bash[22596]: cluster 2024-09-13T00:28:51.283561+0000 mgr.y (mgr.14150) 331 : cluster [DBG] pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:53.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:52 smithi077 bash[32040]: cluster 2024-09-13T00:28:51.283561+0000 mgr.y (mgr.14150) 331 : cluster [DBG] pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:53.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:52 smithi088 bash[24895]: cluster 2024-09-13T00:28:51.283561+0000 mgr.y (mgr.14150) 331 : cluster [DBG] pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:53.881 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:28:55.182 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:54 smithi077 bash[22596]: cluster 2024-09-13T00:28:53.284155+0000 mgr.y (mgr.14150) 332 : cluster [DBG] pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:55.182 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:54 smithi077 bash[32040]: cluster 2024-09-13T00:28:53.284155+0000 mgr.y (mgr.14150) 332 : cluster [DBG] pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:55.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:54 smithi088 bash[24895]: cluster 2024-09-13T00:28:53.284155+0000 mgr.y (mgr.14150) 332 : cluster [DBG] pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:57.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:56 smithi077 bash[22596]: cluster 2024-09-13T00:28:55.284503+0000 mgr.y (mgr.14150) 333 : cluster [DBG] pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:57.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:56 smithi077 bash[32040]: cluster 2024-09-13T00:28:55.284503+0000 mgr.y (mgr.14150) 333 : cluster [DBG] pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:57.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:56 smithi088 bash[24895]: cluster 2024-09-13T00:28:55.284503+0000 mgr.y (mgr.14150) 333 : cluster [DBG] pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:57.644 INFO:teuthology.orchestra.run.smithi077.stdout:pg_num: 1 2024-09-13T00:28:58.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:57 smithi077 bash[22596]: audit 2024-09-13T00:28:57.648028+0000 mon.a (mon.0) 608 : audit [DBG] from='client.? 172.21.15.77:0/1289362190' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T00:28:58.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:57 smithi077 bash[32040]: audit 2024-09-13T00:28:57.648028+0000 mon.a (mon.0) 608 : audit [DBG] from='client.? 172.21.15.77:0/1289362190' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T00:28:58.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:57 smithi088 bash[24895]: audit 2024-09-13T00:28:57.648028+0000 mon.a (mon.0) 608 : audit [DBG] from='client.? 172.21.15.77:0/1289362190' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-13T00:28:58.417 INFO:tasks.cephadm:Adding prometheus.a on smithi088 2024-09-13T00:28:58.417 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch apply prometheus '1;smithi088=a' 2024-09-13T00:28:59.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:28:58 smithi077 bash[22596]: cluster 2024-09-13T00:28:57.285048+0000 mgr.y (mgr.14150) 334 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:59.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:28:58 smithi077 bash[32040]: cluster 2024-09-13T00:28:57.285048+0000 mgr.y (mgr.14150) 334 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:28:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:28:58 smithi088 bash[24895]: cluster 2024-09-13T00:28:57.285048+0000 mgr.y (mgr.14150) 334 : cluster [DBG] pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:01.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:00 smithi077 bash[22596]: cluster 2024-09-13T00:28:59.285648+0000 mgr.y (mgr.14150) 335 : cluster [DBG] pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:01.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:00 smithi077 bash[32040]: cluster 2024-09-13T00:28:59.285648+0000 mgr.y (mgr.14150) 335 : cluster [DBG] pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:01.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:00 smithi088 bash[24895]: cluster 2024-09-13T00:28:59.285648+0000 mgr.y (mgr.14150) 335 : cluster [DBG] pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:02.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:01 smithi077 bash[22596]: cluster 2024-09-13T00:29:01.286235+0000 mgr.y (mgr.14150) 336 : cluster [DBG] pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:02.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:01 smithi077 bash[32040]: cluster 2024-09-13T00:29:01.286235+0000 mgr.y (mgr.14150) 336 : cluster [DBG] pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:02.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:01 smithi088 bash[24895]: cluster 2024-09-13T00:29:01.286235+0000 mgr.y (mgr.14150) 336 : cluster [DBG] pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:03.166 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T00:29:04.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:04 smithi088 bash[24895]: cluster 2024-09-13T00:29:03.286839+0000 mgr.y (mgr.14150) 337 : cluster [DBG] pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:04.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:04 smithi077 bash[22596]: cluster 2024-09-13T00:29:03.286839+0000 mgr.y (mgr.14150) 337 : cluster [DBG] pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:04.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:04 smithi077 bash[32040]: cluster 2024-09-13T00:29:03.286839+0000 mgr.y (mgr.14150) 337 : cluster [DBG] pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:05.977 INFO:teuthology.orchestra.run.smithi088.stdout:Scheduled prometheus update... 2024-09-13T00:29:06.555 DEBUG:teuthology.orchestra.run.smithi088:prometheus.a> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@prometheus.a.service 2024-09-13T00:29:06.557 INFO:tasks.cephadm:Adding node-exporter.a on smithi077 2024-09-13T00:29:06.557 INFO:tasks.cephadm:Adding node-exporter.b on smithi088 2024-09-13T00:29:06.558 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch apply node-exporter '2;smithi077=a;smithi088=b' 2024-09-13T00:29:06.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:06 smithi088 bash[24895]: cluster 2024-09-13T00:29:05.287424+0000 mgr.y (mgr.14150) 338 : cluster [DBG] pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:06.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:06 smithi088 bash[24895]: audit 2024-09-13T00:29:05.979945+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:06.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:06 smithi088 bash[24895]: audit 2024-09-13T00:29:05.981586+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:29:06.676 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:06 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:06.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:06 smithi077 bash[22596]: cluster 2024-09-13T00:29:05.287424+0000 mgr.y (mgr.14150) 338 : cluster [DBG] pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:06.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:06 smithi077 bash[22596]: audit 2024-09-13T00:29:05.979945+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:06.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:06 smithi077 bash[22596]: audit 2024-09-13T00:29:05.981586+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:29:06.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:06 smithi077 bash[32040]: cluster 2024-09-13T00:29:05.287424+0000 mgr.y (mgr.14150) 338 : cluster [DBG] pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:06.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:06 smithi077 bash[32040]: audit 2024-09-13T00:29:05.979945+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:06.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:06 smithi077 bash[32040]: audit 2024-09-13T00:29:05.981586+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:29:07.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:07 smithi088 bash[24895]: audit 2024-09-13T00:29:05.970024+0000 mgr.y (mgr.14150) 339 : audit [DBG] from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi088=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:29:07.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:07 smithi088 bash[24895]: cephadm 2024-09-13T00:29:05.972923+0000 mgr.y (mgr.14150) 340 : cephadm [INF] Saving service prometheus spec with placement smithi088=a;count:1 2024-09-13T00:29:07.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:07 smithi088 bash[24895]: audit 2024-09-13T00:29:06.419703+0000 mon.a (mon.0) 611 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:29:07.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:07 smithi088 bash[24895]: audit 2024-09-13T00:29:06.421380+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:29:07.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:07 smithi088 bash[24895]: audit 2024-09-13T00:29:06.430296+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:07.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:07 smithi088 bash[24895]: cephadm 2024-09-13T00:29:06.463887+0000 mgr.y (mgr.14150) 341 : cephadm [INF] Deploying daemon prometheus.a on smithi088 2024-09-13T00:29:07.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:07 smithi077 bash[22596]: audit 2024-09-13T00:29:05.970024+0000 mgr.y (mgr.14150) 339 : audit [DBG] from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi088=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:29:07.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:07 smithi077 bash[22596]: cephadm 2024-09-13T00:29:05.972923+0000 mgr.y (mgr.14150) 340 : cephadm [INF] Saving service prometheus spec with placement smithi088=a;count:1 2024-09-13T00:29:07.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:07 smithi077 bash[22596]: audit 2024-09-13T00:29:06.419703+0000 mon.a (mon.0) 611 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:29:07.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:07 smithi077 bash[22596]: audit 2024-09-13T00:29:06.421380+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:29:07.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:07 smithi077 bash[22596]: audit 2024-09-13T00:29:06.430296+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:07.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:07 smithi077 bash[22596]: cephadm 2024-09-13T00:29:06.463887+0000 mgr.y (mgr.14150) 341 : cephadm [INF] Deploying daemon prometheus.a on smithi088 2024-09-13T00:29:07.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:07 smithi077 bash[32040]: audit 2024-09-13T00:29:05.970024+0000 mgr.y (mgr.14150) 339 : audit [DBG] from='client.24277 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi088=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:29:07.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:07 smithi077 bash[32040]: cephadm 2024-09-13T00:29:05.972923+0000 mgr.y (mgr.14150) 340 : cephadm [INF] Saving service prometheus spec with placement smithi088=a;count:1 2024-09-13T00:29:07.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:07 smithi077 bash[32040]: audit 2024-09-13T00:29:06.419703+0000 mon.a (mon.0) 611 : audit [DBG] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:29:07.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:07 smithi077 bash[32040]: audit 2024-09-13T00:29:06.421380+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:29:07.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:07 smithi077 bash[32040]: audit 2024-09-13T00:29:06.430296+0000 mon.a (mon.0) 613 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:07.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:07 smithi077 bash[32040]: cephadm 2024-09-13T00:29:06.463887+0000 mgr.y (mgr.14150) 341 : cephadm [INF] Deploying daemon prometheus.a on smithi088 2024-09-13T00:29:08.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:08 smithi077 bash[22596]: cluster 2024-09-13T00:29:07.288007+0000 mgr.y (mgr.14150) 342 : cluster [DBG] pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:08.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:08 smithi077 bash[32040]: cluster 2024-09-13T00:29:07.288007+0000 mgr.y (mgr.14150) 342 : cluster [DBG] pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:08.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:08 smithi088 bash[24895]: cluster 2024-09-13T00:29:07.288007+0000 mgr.y (mgr.14150) 342 : cluster [DBG] pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:10.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:10 smithi077 bash[22596]: cluster 2024-09-13T00:29:09.288641+0000 mgr.y (mgr.14150) 343 : cluster [DBG] pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:10.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:10 smithi077 bash[32040]: cluster 2024-09-13T00:29:09.288641+0000 mgr.y (mgr.14150) 343 : cluster [DBG] pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:10.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:10 smithi088 bash[24895]: cluster 2024-09-13T00:29:09.288641+0000 mgr.y (mgr.14150) 343 : cluster [DBG] pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:11.343 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T00:29:12.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:12 smithi077 bash[22596]: cluster 2024-09-13T00:29:11.289264+0000 mgr.y (mgr.14150) 344 : cluster [DBG] pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:12.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:12 smithi077 bash[32040]: cluster 2024-09-13T00:29:11.289264+0000 mgr.y (mgr.14150) 344 : cluster [DBG] pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:12.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:12 smithi088 bash[24895]: cluster 2024-09-13T00:29:11.289264+0000 mgr.y (mgr.14150) 344 : cluster [DBG] pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:14.728 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:14 smithi088 bash[24895]: cluster 2024-09-13T00:29:13.289801+0000 mgr.y (mgr.14150) 345 : cluster [DBG] pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:14.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:14 smithi077 bash[22596]: cluster 2024-09-13T00:29:13.289801+0000 mgr.y (mgr.14150) 345 : cluster [DBG] pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:14.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:14 smithi077 bash[32040]: cluster 2024-09-13T00:29:13.289801+0000 mgr.y (mgr.14150) 345 : cluster [DBG] pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:16.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:16 smithi077 bash[22596]: cluster 2024-09-13T00:29:15.290349+0000 mgr.y (mgr.14150) 346 : cluster [DBG] pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:16.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:16 smithi077 bash[32040]: cluster 2024-09-13T00:29:15.290349+0000 mgr.y (mgr.14150) 346 : cluster [DBG] pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:16.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:16 smithi088 bash[24895]: cluster 2024-09-13T00:29:15.290349+0000 mgr.y (mgr.14150) 346 : cluster [DBG] pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:17.290 INFO:teuthology.orchestra.run.smithi088.stdout:Scheduled node-exporter update... 2024-09-13T00:29:18.334 DEBUG:teuthology.orchestra.run.smithi077:node-exporter.a> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@node-exporter.a.service 2024-09-13T00:29:18.337 DEBUG:teuthology.orchestra.run.smithi088:node-exporter.b> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@node-exporter.b.service 2024-09-13T00:29:18.340 INFO:tasks.cephadm:Adding alertmanager.a on smithi077 2024-09-13T00:29:18.340 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch apply alertmanager '1;smithi077=a' 2024-09-13T00:29:18.562 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:18 smithi088 bash[24895]: audit 2024-09-13T00:29:17.283295+0000 mgr.y (mgr.14150) 347 : audit [DBG] from='client.24283 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi077=a;smithi088=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:29:18.563 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:18 smithi088 bash[24895]: cephadm 2024-09-13T00:29:17.286337+0000 mgr.y (mgr.14150) 348 : cephadm [INF] Saving service node-exporter spec with placement smithi077=a;smithi088=b;count:2 2024-09-13T00:29:18.563 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:18 smithi088 bash[24895]: cluster 2024-09-13T00:29:17.290858+0000 mgr.y (mgr.14150) 349 : cluster [DBG] pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:18.563 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:18 smithi088 bash[24895]: audit 2024-09-13T00:29:17.293714+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:18.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:18 smithi077 bash[22596]: audit 2024-09-13T00:29:17.283295+0000 mgr.y (mgr.14150) 347 : audit [DBG] from='client.24283 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi077=a;smithi088=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:29:18.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:18 smithi077 bash[22596]: cephadm 2024-09-13T00:29:17.286337+0000 mgr.y (mgr.14150) 348 : cephadm [INF] Saving service node-exporter spec with placement smithi077=a;smithi088=b;count:2 2024-09-13T00:29:18.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:18 smithi077 bash[22596]: cluster 2024-09-13T00:29:17.290858+0000 mgr.y (mgr.14150) 349 : cluster [DBG] pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:18.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:18 smithi077 bash[22596]: audit 2024-09-13T00:29:17.293714+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:18.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:18 smithi077 bash[32040]: audit 2024-09-13T00:29:17.283295+0000 mgr.y (mgr.14150) 347 : audit [DBG] from='client.24283 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi077=a;smithi088=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:29:18.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:18 smithi077 bash[32040]: cephadm 2024-09-13T00:29:17.286337+0000 mgr.y (mgr.14150) 348 : cephadm [INF] Saving service node-exporter spec with placement smithi077=a;smithi088=b;count:2 2024-09-13T00:29:18.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:18 smithi077 bash[32040]: cluster 2024-09-13T00:29:17.290858+0000 mgr.y (mgr.14150) 349 : cluster [DBG] pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:18.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:18 smithi077 bash[32040]: audit 2024-09-13T00:29:17.293714+0000 mon.a (mon.0) 614 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:20.485 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:20 smithi088 bash[24895]: cluster 2024-09-13T00:29:19.291445+0000 mgr.y (mgr.14150) 350 : cluster [DBG] pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:20.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:20 smithi077 bash[22596]: cluster 2024-09-13T00:29:19.291445+0000 mgr.y (mgr.14150) 350 : cluster [DBG] pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:20.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:20 smithi077 bash[32040]: cluster 2024-09-13T00:29:19.291445+0000 mgr.y (mgr.14150) 350 : cluster [DBG] pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:21.100 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:29:20 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:21.101 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:29:20 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:21.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:21 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:21.101 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:29:20 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:21.102 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:21 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:21.102 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 00:29:20 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:21.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:21 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:21.351 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:21 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:21.351 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:29:21 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:21.351 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 00:29:21 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:21.351 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:29:21 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:21.352 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:29:21 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:21.352 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:21 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:21.352 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:21 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:21.352 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:21 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:21.352 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:21 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:21.352 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:21 smithi088 systemd[1]: Started Ceph prometheus.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:29:22.546 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[25838]: ignoring --setuser ceph since I am not root 2024-09-13T00:29:22.546 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[25838]: ignoring --setgroup ceph since I am not root 2024-09-13T00:29:22.547 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[25838]: debug 2024-09-13T00:29:22.436+0000 7f1aa4635640 1 -- 172.21.15.88:0/3433755082 <== mon.1 v2:172.21.15.88:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b7ebe44340 con 0x55b7ebe50800 2024-09-13T00:29:22.547 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[24895]: cluster 2024-09-13T00:29:21.292009+0000 mgr.y (mgr.14150) 351 : cluster [DBG] pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:22.547 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[24895]: audit 2024-09-13T00:29:21.375263+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:22.547 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[24895]: audit 2024-09-13T00:29:21.384281+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:22.547 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[24895]: audit 2024-09-13T00:29:21.392503+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:22.547 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[24895]: audit 2024-09-13T00:29:21.400332+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T00:29:22.548 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.372Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T00:29:22.548 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.372Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T00:29:22.548 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.372Z caller=main.go:570 level=info host_details="(Linux 5.15.0-121-generic #131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024 x86_64 smithi088 (none))" 2024-09-13T00:29:22.548 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.372Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T00:29:22.548 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.373Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T00:29:22.548 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.374Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T00:29:22.548 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.374Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T00:29:22.549 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.376Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T00:29:22.549 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.376Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T00:29:22.549 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.380Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T00:29:22.549 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.380Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.598µs 2024-09-13T00:29:22.549 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.380Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T00:29:22.549 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.380Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-09-13T00:29:22.549 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.380Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=29.365µs wal_replay_duration=365.036µs wbl_replay_duration=129ns total_replay_duration=424.036µs 2024-09-13T00:29:22.549 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.381Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T00:29:22.549 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.381Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T00:29:22.549 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.381Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T00:29:22.550 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.399Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.162616ms db_storage=877ns remote_storage=1.292µs web_handler=408ns query_engine=707ns scrape=281.802µs scrape_sd=21.131µs notify=958ns notify_sd=1.505µs rules=16.601693ms tracing=7.28µs 2024-09-13T00:29:22.550 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.399Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T00:29:22.550 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[49366]: ts=2024-09-13T00:29:22.399Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T00:29:22.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:22 smithi077 bash[22596]: cluster 2024-09-13T00:29:21.292009+0000 mgr.y (mgr.14150) 351 : cluster [DBG] pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:22.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:22 smithi077 bash[22596]: audit 2024-09-13T00:29:21.375263+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:22.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:22 smithi077 bash[22596]: audit 2024-09-13T00:29:21.384281+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:22.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:22 smithi077 bash[22596]: audit 2024-09-13T00:29:21.392503+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:22.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:22 smithi077 bash[22596]: audit 2024-09-13T00:29:21.400332+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T00:29:22.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:22 smithi077 bash[22845]: ignoring --setuser ceph since I am not root 2024-09-13T00:29:22.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:22 smithi077 bash[22845]: ignoring --setgroup ceph since I am not root 2024-09-13T00:29:22.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:22 smithi077 bash[22845]: debug 2024-09-13T00:29:22.470+0000 7f8ab9cba640 1 -- 172.21.15.77:0/3628489433 <== mon.0 v2:172.21.15.77:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56126a216340 con 0x56126a223000 2024-09-13T00:29:22.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:22 smithi077 bash[22845]: debug 2024-09-13T00:29:22.578+0000 7f8abc514ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T00:29:22.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:22 smithi077 bash[32040]: cluster 2024-09-13T00:29:21.292009+0000 mgr.y (mgr.14150) 351 : cluster [DBG] pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:22.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:22 smithi077 bash[32040]: audit 2024-09-13T00:29:21.375263+0000 mon.a (mon.0) 615 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:22.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:22 smithi077 bash[32040]: audit 2024-09-13T00:29:21.384281+0000 mon.a (mon.0) 616 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:22.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:22 smithi077 bash[32040]: audit 2024-09-13T00:29:21.392503+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' 2024-09-13T00:29:22.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:22 smithi077 bash[32040]: audit 2024-09-13T00:29:21.400332+0000 mon.a (mon.0) 618 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-13T00:29:22.850 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:22 smithi088 bash[25838]: debug 2024-09-13T00:29:22.548+0000 7f1aa6e8fec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T00:29:23.707 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:23 smithi088 bash[24895]: audit 2024-09-13T00:29:22.408147+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T00:29:23.708 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:23 smithi088 bash[24895]: cluster 2024-09-13T00:29:22.408257+0000 mon.a (mon.0) 620 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-09-13T00:29:23.765 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:23 smithi077 bash[22596]: audit 2024-09-13T00:29:22.408147+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T00:29:23.765 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:23 smithi077 bash[22596]: cluster 2024-09-13T00:29:22.408257+0000 mon.a (mon.0) 620 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-09-13T00:29:23.765 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:23 smithi077 bash[32040]: audit 2024-09-13T00:29:22.408147+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.14150 172.21.15.77:0/2948783267' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-13T00:29:23.765 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:23 smithi077 bash[32040]: cluster 2024-09-13T00:29:22.408257+0000 mon.a (mon.0) 620 : cluster [DBG] mgrmap e16: y(active, since 9m), standbys: x 2024-09-13T00:29:24.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:23 smithi077 bash[22845]: debug 2024-09-13T00:29:23.766+0000 7f8abc514ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T00:29:24.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:23 smithi077 bash[22845]: debug 2024-09-13T00:29:23.830+0000 7f8abc514ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T00:29:24.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:23 smithi077 bash[22845]: debug 2024-09-13T00:29:23.982+0000 7f8abc514ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T00:29:24.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:23 smithi088 bash[25838]: debug 2024-09-13T00:29:23.708+0000 7f1aa6e8fec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T00:29:24.101 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:23 smithi088 bash[25838]: debug 2024-09-13T00:29:23.768+0000 7f1aa6e8fec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T00:29:24.101 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:23 smithi088 bash[25838]: debug 2024-09-13T00:29:23.920+0000 7f1aa6e8fec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T00:29:24.138 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T00:29:24.838 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:24 smithi088 bash[25838]: debug 2024-09-13T00:29:24.548+0000 7f1aa6e8fec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T00:29:24.839 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:24 smithi088 bash[25838]: debug 2024-09-13T00:29:24.616+0000 7f1aa6e8fec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T00:29:24.839 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:24 smithi088 bash[25838]: debug 2024-09-13T00:29:24.684+0000 7f1aa6e8fec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T00:29:24.894 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:24 smithi077 bash[22845]: debug 2024-09-13T00:29:24.606+0000 7f8abc514ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T00:29:24.895 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:24 smithi077 bash[22845]: debug 2024-09-13T00:29:24.674+0000 7f8abc514ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T00:29:24.895 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:24 smithi077 bash[22845]: debug 2024-09-13T00:29:24.738+0000 7f8abc514ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T00:29:25.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:24 smithi088 bash[25838]: debug 2024-09-13T00:29:24.840+0000 7f1aa6e8fec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T00:29:25.101 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:24 smithi088 bash[25838]: debug 2024-09-13T00:29:24.916+0000 7f1aa6e8fec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T00:29:25.101 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:24 smithi088 bash[25838]: debug 2024-09-13T00:29:24.980+0000 7f1aa6e8fec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T00:29:25.262 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:24 smithi077 bash[22845]: debug 2024-09-13T00:29:24.898+0000 7f8abc514ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T00:29:25.262 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:24 smithi077 bash[22845]: debug 2024-09-13T00:29:24.970+0000 7f8abc514ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T00:29:25.262 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:25 smithi077 bash[22845]: debug 2024-09-13T00:29:25.038+0000 7f8abc514ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T00:29:25.490 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:25 smithi088 bash[25838]: debug 2024-09-13T00:29:25.208+0000 7f1aa6e8fec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T00:29:25.490 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:25 smithi088 bash[25838]: debug 2024-09-13T00:29:25.272+0000 7f1aa6e8fec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T00:29:25.490 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:25 smithi088 bash[25838]: debug 2024-09-13T00:29:25.444+0000 7f1aa6e8fec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T00:29:25.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:25 smithi077 bash[22845]: debug 2024-09-13T00:29:25.262+0000 7f8abc514ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T00:29:25.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:25 smithi077 bash[22845]: debug 2024-09-13T00:29:25.330+0000 7f8abc514ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T00:29:25.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:25 smithi077 bash[22845]: debug 2024-09-13T00:29:25.494+0000 7f8abc514ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T00:29:25.758 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:25 smithi088 bash[25838]: debug 2024-09-13T00:29:25.524+0000 7f1aa6e8fec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T00:29:25.960 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:25 smithi077 bash[22845]: debug 2024-09-13T00:29:25.570+0000 7f8abc514ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T00:29:25.960 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:25 smithi077 bash[22845]: debug 2024-09-13T00:29:25.802+0000 7f8abc514ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:29:26.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:25 smithi088 bash[25838]: debug 2024-09-13T00:29:25.756+0000 7f1aa6e8fec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:29:26.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:25 smithi088 bash[25838]: debug 2024-09-13T00:29:25.916+0000 7f1aa6e8fec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T00:29:26.101 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:25 smithi088 bash[25838]: debug 2024-09-13T00:29:25.992+0000 7f1aa6e8fec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T00:29:26.277 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:25 smithi077 bash[22845]: debug 2024-09-13T00:29:25.962+0000 7f8abc514ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T00:29:26.277 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:26 smithi077 bash[22845]: debug 2024-09-13T00:29:26.038+0000 7f8abc514ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T00:29:26.277 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:26 smithi077 bash[22845]: debug 2024-09-13T00:29:26.190+0000 7f8abc514ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:29:26.600 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:26 smithi088 bash[25838]: debug 2024-09-13T00:29:26.148+0000 7f1aa6e8fec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:29:27.224 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:26 smithi088 bash[25838]: debug 2024-09-13T00:29:26.940+0000 7f1aa6e8fec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T00:29:27.224 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[25838]: debug 2024-09-13T00:29:27.020+0000 7f1aa6e8fec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T00:29:27.238 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:26 smithi077 bash[22845]: debug 2024-09-13T00:29:26.970+0000 7f8abc514ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T00:29:27.238 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22845]: debug 2024-09-13T00:29:27.042+0000 7f8abc514ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T00:29:27.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22845]: debug 2024-09-13T00:29:27.238+0000 7f8abc514ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T00:29:27.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22845]: debug 2024-09-13T00:29:27.310+0000 7f8abc514ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T00:29:27.600 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[25838]: debug 2024-09-13T00:29:27.224+0000 7f1aa6e8fec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T00:29:27.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[25838]: debug 2024-09-13T00:29:27.296+0000 7f1aa6e8fec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T00:29:27.946 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22845]: debug 2024-09-13T00:29:27.626+0000 7f8abc514ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T00:29:27.946 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22845]: debug 2024-09-13T00:29:27.702+0000 7f8abc514ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T00:29:27.946 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22845]: [13/Sep/2024:00:29:27] ENGINE Bus STARTING 2024-09-13T00:29:27.946 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22845]: CherryPy Checker: 2024-09-13T00:29:27.947 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22845]: The Application mounted at '' has an empty config. 2024-09-13T00:29:27.947 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22596]: cluster 2024-09-13T00:29:27.692221+0000 mon.a (mon.0) 621 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T00:29:27.947 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22596]: cluster 2024-09-13T00:29:27.692362+0000 mon.a (mon.0) 622 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:29:27.947 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22596]: audit 2024-09-13T00:29:27.693809+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.88:0/1111085537' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:29:27.947 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22596]: audit 2024-09-13T00:29:27.695077+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.88:0/1111085537' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:29:27.947 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22596]: audit 2024-09-13T00:29:27.697608+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.88:0/1111085537' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:29:27.947 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22596]: audit 2024-09-13T00:29:27.698181+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.88:0/1111085537' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:29:27.947 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22596]: cluster 2024-09-13T00:29:27.728314+0000 mon.a (mon.0) 623 : cluster [INF] Active manager daemon y restarted 2024-09-13T00:29:27.947 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22596]: cluster 2024-09-13T00:29:27.728994+0000 mon.a (mon.0) 624 : cluster [INF] Activating manager daemon y 2024-09-13T00:29:27.948 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[32040]: cluster 2024-09-13T00:29:27.692221+0000 mon.a (mon.0) 621 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T00:29:27.948 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[32040]: cluster 2024-09-13T00:29:27.692362+0000 mon.a (mon.0) 622 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:29:27.948 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[32040]: audit 2024-09-13T00:29:27.693809+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.88:0/1111085537' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:29:27.948 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[32040]: audit 2024-09-13T00:29:27.695077+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.88:0/1111085537' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:29:27.948 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[32040]: audit 2024-09-13T00:29:27.697608+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.88:0/1111085537' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:29:27.948 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[32040]: audit 2024-09-13T00:29:27.698181+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.88:0/1111085537' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:29:27.948 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[32040]: cluster 2024-09-13T00:29:27.728314+0000 mon.a (mon.0) 623 : cluster [INF] Active manager daemon y restarted 2024-09-13T00:29:27.948 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[32040]: cluster 2024-09-13T00:29:27.728994+0000 mon.a (mon.0) 624 : cluster [INF] Activating manager daemon y 2024-09-13T00:29:27.954 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[25838]: debug 2024-09-13T00:29:27.608+0000 7f1aa6e8fec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T00:29:27.954 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[25838]: debug 2024-09-13T00:29:27.684+0000 7f1aa6e8fec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T00:29:27.954 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[25838]: [13/Sep/2024:00:29:27] ENGINE Bus STARTING 2024-09-13T00:29:27.954 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[25838]: CherryPy Checker: 2024-09-13T00:29:27.954 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[25838]: The Application mounted at '' has an empty config. 2024-09-13T00:29:27.954 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[25838]: [13/Sep/2024:00:29:27] ENGINE Serving on http://:::9283 2024-09-13T00:29:27.954 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[25838]: [13/Sep/2024:00:29:27] ENGINE Bus STARTED 2024-09-13T00:29:27.955 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[24895]: cluster 2024-09-13T00:29:27.692221+0000 mon.a (mon.0) 621 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T00:29:27.955 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[24895]: cluster 2024-09-13T00:29:27.692362+0000 mon.a (mon.0) 622 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:29:27.955 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[24895]: audit 2024-09-13T00:29:27.693809+0000 mon.b (mon.1) 25 : audit [DBG] from='mgr.? 172.21.15.88:0/1111085537' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:29:27.955 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[24895]: audit 2024-09-13T00:29:27.695077+0000 mon.b (mon.1) 26 : audit [DBG] from='mgr.? 172.21.15.88:0/1111085537' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:29:27.955 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[24895]: audit 2024-09-13T00:29:27.697608+0000 mon.b (mon.1) 27 : audit [DBG] from='mgr.? 172.21.15.88:0/1111085537' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:29:27.955 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[24895]: audit 2024-09-13T00:29:27.698181+0000 mon.b (mon.1) 28 : audit [DBG] from='mgr.? 172.21.15.88:0/1111085537' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:29:27.955 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[24895]: cluster 2024-09-13T00:29:27.728314+0000 mon.a (mon.0) 623 : cluster [INF] Active manager daemon y restarted 2024-09-13T00:29:27.955 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:27 smithi088 bash[24895]: cluster 2024-09-13T00:29:27.728994+0000 mon.a (mon.0) 624 : cluster [INF] Activating manager daemon y 2024-09-13T00:29:28.277 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22845]: [13/Sep/2024:00:29:27] ENGINE Serving on http://:::9283 2024-09-13T00:29:28.277 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:27 smithi077 bash[22845]: [13/Sep/2024:00:29:27] ENGINE Bus STARTED 2024-09-13T00:29:28.781 INFO:teuthology.orchestra.run.smithi088.stdout:Scheduled alertmanager update... 2024-09-13T00:29:29.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: cluster 2024-09-13T00:29:27.742551+0000 mon.a (mon.0) 625 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2024-09-13T00:29:29.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: cluster 2024-09-13T00:29:27.743311+0000 mon.a (mon.0) 626 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0146194s), standbys: x 2024-09-13T00:29:29.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.743479+0000 mon.c (mon.2) 13 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:29:29.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.743761+0000 mon.c (mon.2) 14 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:29:29.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.744058+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:29:29.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.748785+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T00:29:29.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.748866+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T00:29:29.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.748932+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:29:29.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.749001+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:29:29.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.749062+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:29:29.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.749128+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:29:29.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.749189+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:29:29.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.749252+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:29:29.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.749316+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:29:29.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.749379+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:29:29.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.749453+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T00:29:29.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.749511+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T00:29:29.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.749570+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T00:29:29.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: cluster 2024-09-13T00:29:27.782801+0000 mon.a (mon.0) 627 : cluster [INF] Manager daemon y is now available 2024-09-13T00:29:29.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.799257+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:29:29.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.811870+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:29.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.823331+0000 mon.c (mon.2) 30 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:29:29.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.823723+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:29:29.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.823779+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:29.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.829340+0000 mon.c (mon.2) 32 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:29:29.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[32040]: audit 2024-09-13T00:29:27.829688+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:29:29.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: cluster 2024-09-13T00:29:27.742551+0000 mon.a (mon.0) 625 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2024-09-13T00:29:29.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: cluster 2024-09-13T00:29:27.743311+0000 mon.a (mon.0) 626 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0146194s), standbys: x 2024-09-13T00:29:29.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.743479+0000 mon.c (mon.2) 13 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:29:29.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.743761+0000 mon.c (mon.2) 14 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:29:29.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.744058+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:29:29.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.748785+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T00:29:29.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.748866+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T00:29:29.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.748932+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:29:29.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.749001+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:29:29.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.749062+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:29:29.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.749128+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:29:29.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.749189+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:29:29.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.749252+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:29:29.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.749316+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:29:29.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.749379+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:29:29.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.749453+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T00:29:29.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.749511+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T00:29:29.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.749570+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T00:29:29.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: cluster 2024-09-13T00:29:27.782801+0000 mon.a (mon.0) 627 : cluster [INF] Manager daemon y is now available 2024-09-13T00:29:29.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.799257+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:29:29.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.811870+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:29.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.823331+0000 mon.c (mon.2) 30 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:29:29.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.823723+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:29:29.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.823779+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:29.034 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.829340+0000 mon.c (mon.2) 32 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:29:29.034 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:28 smithi077 bash[22596]: audit 2024-09-13T00:29:27.829688+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:29:29.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: cluster 2024-09-13T00:29:27.742551+0000 mon.a (mon.0) 625 : cluster [DBG] osdmap e51: 8 total, 8 up, 8 in 2024-09-13T00:29:29.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: cluster 2024-09-13T00:29:27.743311+0000 mon.a (mon.0) 626 : cluster [DBG] mgrmap e17: y(active, starting, since 0.0146194s), standbys: x 2024-09-13T00:29:29.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.743479+0000 mon.c (mon.2) 13 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:29:29.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.743761+0000 mon.c (mon.2) 14 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:29:29.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.744058+0000 mon.c (mon.2) 15 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:29:29.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.748785+0000 mon.c (mon.2) 16 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T00:29:29.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.748866+0000 mon.c (mon.2) 17 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T00:29:29.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.748932+0000 mon.c (mon.2) 18 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:29:29.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.749001+0000 mon.c (mon.2) 19 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:29:29.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.749062+0000 mon.c (mon.2) 20 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:29:29.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.749128+0000 mon.c (mon.2) 21 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:29:29.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.749189+0000 mon.c (mon.2) 22 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:29:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.749252+0000 mon.c (mon.2) 23 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:29:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.749316+0000 mon.c (mon.2) 24 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:29:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.749379+0000 mon.c (mon.2) 25 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:29:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.749453+0000 mon.c (mon.2) 26 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T00:29:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.749511+0000 mon.c (mon.2) 27 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T00:29:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.749570+0000 mon.c (mon.2) 28 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T00:29:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: cluster 2024-09-13T00:29:27.782801+0000 mon.a (mon.0) 627 : cluster [INF] Manager daemon y is now available 2024-09-13T00:29:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.799257+0000 mon.c (mon.2) 29 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:29:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.811870+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.823331+0000 mon.c (mon.2) 30 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:29:29.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.823723+0000 mon.a (mon.0) 629 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:29:29.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.823779+0000 mon.c (mon.2) 31 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:29.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.829340+0000 mon.c (mon.2) 32 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:29:29.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:28 smithi088 bash[24895]: audit 2024-09-13T00:29:27.829688+0000 mon.a (mon.0) 630 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:29:29.409 DEBUG:teuthology.orchestra.run.smithi077:alertmanager.a> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@alertmanager.a.service 2024-09-13T00:29:29.412 INFO:tasks.cephadm:Adding grafana.a on smithi088 2024-09-13T00:29:29.412 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph orch apply grafana '1;smithi088=a' 2024-09-13T00:29:29.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: debug 2024-09-13T00:29:29.210+0000 7f8a5f286640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:00:29:29] ENGINE Error in HTTPServer.serve 2024-09-13T00:29:29.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: Traceback (most recent call last): 2024-09-13T00:29:29.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:29:29.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: self._connections.run(self.expiration_interval) 2024-09-13T00:29:29.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:29:29.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: self._run(expiration_interval) 2024-09-13T00:29:29.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:29:29.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:29:29.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:29:29.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:29:29.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:29:29.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: s = self.context.wrap_socket( 2024-09-13T00:29:29.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:29:29.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: return self.sslsocket_class._create( 2024-09-13T00:29:29.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:29:29.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: self.do_handshake() 2024-09-13T00:29:29.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:29:29.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: self._sslobj.do_handshake() 2024-09-13T00:29:29.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22845]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:29:30.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: cluster 2024-09-13T00:29:28.763753+0000 mon.a (mon.0) 631 : cluster [DBG] mgrmap e18: y(active, since 1.03506s), standbys: x 2024-09-13T00:29:30.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: cephadm 2024-09-13T00:29:28.777444+0000 mgr.y (mgr.14397) 2 : cephadm [INF] Saving service alertmanager spec with placement smithi077=a;count:1 2024-09-13T00:29:30.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: cluster 2024-09-13T00:29:28.782937+0000 mgr.y (mgr.14397) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:30.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: audit 2024-09-13T00:29:28.784056+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:30.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: cephadm 2024-09-13T00:29:29.089042+0000 mgr.y (mgr.14397) 4 : cephadm [INF] [13/Sep/2024:00:29:29] ENGINE Bus STARTING 2024-09-13T00:29:30.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: cephadm 2024-09-13T00:29:29.208745+0000 mgr.y (mgr.14397) 5 : cephadm [INF] [13/Sep/2024:00:29:29] ENGINE Serving on https://172.21.15.77:7150 2024-09-13T00:29:30.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: cephadm 2024-09-13T00:29:29.208843+0000 mgr.y (mgr.14397) 6 : cephadm [INF] [13/Sep/2024:00:29:29] ENGINE Bus STARTED 2024-09-13T00:29:30.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: cephadm 2024-09-13T00:29:29.210610+0000 mgr.y (mgr.14397) 7 : cephadm [ERR] [13/Sep/2024:00:29:29] ENGINE Error in HTTPServer.serve 2024-09-13T00:29:30.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: Traceback (most recent call last): 2024-09-13T00:29:30.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:29:30.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: self._connections.run(self.expiration_interval) 2024-09-13T00:29:30.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:29:30.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: self._run(expiration_interval) 2024-09-13T00:29:30.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:29:30.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:29:30.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:29:30.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:29:30.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:29:30.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: s = self.context.wrap_socket( 2024-09-13T00:29:30.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:29:30.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: return self.sslsocket_class._create( 2024-09-13T00:29:30.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:29:30.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: self.do_handshake() 2024-09-13T00:29:30.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:29:30.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: self._sslobj.do_handshake() 2024-09-13T00:29:30.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:29:30.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[22596]: audit 2024-09-13T00:29:29.214837+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:30.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: cluster 2024-09-13T00:29:28.763753+0000 mon.a (mon.0) 631 : cluster [DBG] mgrmap e18: y(active, since 1.03506s), standbys: x 2024-09-13T00:29:30.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: cephadm 2024-09-13T00:29:28.777444+0000 mgr.y (mgr.14397) 2 : cephadm [INF] Saving service alertmanager spec with placement smithi077=a;count:1 2024-09-13T00:29:30.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: cluster 2024-09-13T00:29:28.782937+0000 mgr.y (mgr.14397) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:30.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: audit 2024-09-13T00:29:28.784056+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:30.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: cephadm 2024-09-13T00:29:29.089042+0000 mgr.y (mgr.14397) 4 : cephadm [INF] [13/Sep/2024:00:29:29] ENGINE Bus STARTING 2024-09-13T00:29:30.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: cephadm 2024-09-13T00:29:29.208745+0000 mgr.y (mgr.14397) 5 : cephadm [INF] [13/Sep/2024:00:29:29] ENGINE Serving on https://172.21.15.77:7150 2024-09-13T00:29:30.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: cephadm 2024-09-13T00:29:29.208843+0000 mgr.y (mgr.14397) 6 : cephadm [INF] [13/Sep/2024:00:29:29] ENGINE Bus STARTED 2024-09-13T00:29:30.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: cephadm 2024-09-13T00:29:29.210610+0000 mgr.y (mgr.14397) 7 : cephadm [ERR] [13/Sep/2024:00:29:29] ENGINE Error in HTTPServer.serve 2024-09-13T00:29:30.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: Traceback (most recent call last): 2024-09-13T00:29:30.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:29:30.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: self._connections.run(self.expiration_interval) 2024-09-13T00:29:30.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:29:30.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: self._run(expiration_interval) 2024-09-13T00:29:30.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:29:30.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:29:30.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:29:30.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:29:30.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:29:30.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: s = self.context.wrap_socket( 2024-09-13T00:29:30.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:29:30.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: return self.sslsocket_class._create( 2024-09-13T00:29:30.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:29:30.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: self.do_handshake() 2024-09-13T00:29:30.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:29:30.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: self._sslobj.do_handshake() 2024-09-13T00:29:30.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:29:30.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:29 smithi077 bash[32040]: audit 2024-09-13T00:29:29.214837+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:30.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: cluster 2024-09-13T00:29:28.763753+0000 mon.a (mon.0) 631 : cluster [DBG] mgrmap e18: y(active, since 1.03506s), standbys: x 2024-09-13T00:29:30.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: cephadm 2024-09-13T00:29:28.777444+0000 mgr.y (mgr.14397) 2 : cephadm [INF] Saving service alertmanager spec with placement smithi077=a;count:1 2024-09-13T00:29:30.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: cluster 2024-09-13T00:29:28.782937+0000 mgr.y (mgr.14397) 3 : cluster [DBG] pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:30.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: audit 2024-09-13T00:29:28.784056+0000 mon.a (mon.0) 632 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:30.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: cephadm 2024-09-13T00:29:29.089042+0000 mgr.y (mgr.14397) 4 : cephadm [INF] [13/Sep/2024:00:29:29] ENGINE Bus STARTING 2024-09-13T00:29:30.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: cephadm 2024-09-13T00:29:29.208745+0000 mgr.y (mgr.14397) 5 : cephadm [INF] [13/Sep/2024:00:29:29] ENGINE Serving on https://172.21.15.77:7150 2024-09-13T00:29:30.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: cephadm 2024-09-13T00:29:29.208843+0000 mgr.y (mgr.14397) 6 : cephadm [INF] [13/Sep/2024:00:29:29] ENGINE Bus STARTED 2024-09-13T00:29:30.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: cephadm 2024-09-13T00:29:29.210610+0000 mgr.y (mgr.14397) 7 : cephadm [ERR] [13/Sep/2024:00:29:29] ENGINE Error in HTTPServer.serve 2024-09-13T00:29:30.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: Traceback (most recent call last): 2024-09-13T00:29:30.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:29:30.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: self._connections.run(self.expiration_interval) 2024-09-13T00:29:30.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:29:30.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: self._run(expiration_interval) 2024-09-13T00:29:30.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:29:30.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:29:30.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:29:30.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:29:30.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:29:30.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: s = self.context.wrap_socket( 2024-09-13T00:29:30.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:29:30.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: return self.sslsocket_class._create( 2024-09-13T00:29:30.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:29:30.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: self.do_handshake() 2024-09-13T00:29:30.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:29:30.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: self._sslobj.do_handshake() 2024-09-13T00:29:30.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:29:30.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:29 smithi088 bash[24895]: audit 2024-09-13T00:29:29.214837+0000 mon.a (mon.0) 633 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:31.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:30 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:29:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:29:31.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:31 smithi077 bash[22596]: cluster 2024-09-13T00:29:29.748655+0000 mgr.y (mgr.14397) 8 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:31.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:31 smithi077 bash[22596]: cluster 2024-09-13T00:29:30.226823+0000 mon.a (mon.0) 634 : cluster [DBG] mgrmap e19: y(active, since 2s), standbys: x 2024-09-13T00:29:31.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:31 smithi077 bash[32040]: cluster 2024-09-13T00:29:29.748655+0000 mgr.y (mgr.14397) 8 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:31.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:31 smithi077 bash[32040]: cluster 2024-09-13T00:29:30.226823+0000 mon.a (mon.0) 634 : cluster [DBG] mgrmap e19: y(active, since 2s), standbys: x 2024-09-13T00:29:31.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:31 smithi088 bash[24895]: cluster 2024-09-13T00:29:29.748655+0000 mgr.y (mgr.14397) 8 : cluster [DBG] pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:31.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:31 smithi088 bash[24895]: cluster 2024-09-13T00:29:30.226823+0000 mon.a (mon.0) 634 : cluster [DBG] mgrmap e19: y(active, since 2s), standbys: x 2024-09-13T00:29:33.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:33 smithi077 bash[22596]: cluster 2024-09-13T00:29:31.749207+0000 mgr.y (mgr.14397) 9 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:33.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:33 smithi077 bash[22596]: cluster 2024-09-13T00:29:32.242283+0000 mon.a (mon.0) 635 : cluster [DBG] mgrmap e20: y(active, since 4s), standbys: x 2024-09-13T00:29:33.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:33 smithi077 bash[32040]: cluster 2024-09-13T00:29:31.749207+0000 mgr.y (mgr.14397) 9 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:33.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:33 smithi077 bash[32040]: cluster 2024-09-13T00:29:32.242283+0000 mon.a (mon.0) 635 : cluster [DBG] mgrmap e20: y(active, since 4s), standbys: x 2024-09-13T00:29:33.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:33 smithi088 bash[24895]: cluster 2024-09-13T00:29:31.749207+0000 mgr.y (mgr.14397) 9 : cluster [DBG] pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:33.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:33 smithi088 bash[24895]: cluster 2024-09-13T00:29:32.242283+0000 mon.a (mon.0) 635 : cluster [DBG] mgrmap e20: y(active, since 4s), standbys: x 2024-09-13T00:29:33.984 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T00:29:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:33.256209+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:33.265901+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:33.408489+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:33.423298+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:33.843716+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:33.850208+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:33.988872+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:34.001613+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:34.006217+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:34.006914+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:34.009083+0000 mon.c (mon.2) 34 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:34.009747+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:34.011771+0000 mon.c (mon.2) 35 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:34.012398+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:34.014296+0000 mon.c (mon.2) 36 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:34.014916+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:34.025976+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:34.028520+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:29:34.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:34 smithi088 bash[24895]: audit 2024-09-13T00:29:34.030225+0000 mon.c (mon.2) 38 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:29:34.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:33.256209+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:33.265901+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:33.408489+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:33.423298+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:33.843716+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:33.850208+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:33.988872+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:34.001613+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:34.006217+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:34.006914+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:34.009083+0000 mon.c (mon.2) 34 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:34.009747+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:34.011771+0000 mon.c (mon.2) 35 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:34.012398+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:34.014296+0000 mon.c (mon.2) 36 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:34.014916+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:34.025976+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:34.028520+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:29:34.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[22596]: audit 2024-09-13T00:29:34.030225+0000 mon.c (mon.2) 38 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:29:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:33.256209+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:33.265901+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:33.408489+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:33.423298+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:33.843716+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:33.850208+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:33.988872+0000 mon.a (mon.0) 642 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:34.001613+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:34.006217+0000 mon.c (mon.2) 33 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:34.006914+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:34.009083+0000 mon.c (mon.2) 34 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:34.009747+0000 mon.a (mon.0) 645 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:34.011771+0000 mon.c (mon.2) 35 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:34.012398+0000 mon.a (mon.0) 646 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:34.014296+0000 mon.c (mon.2) 36 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:34.014916+0000 mon.a (mon.0) 647 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:29:34.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:34.025976+0000 mon.a (mon.0) 648 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:34.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:34.028520+0000 mon.c (mon.2) 37 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:29:34.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:34 smithi077 bash[32040]: audit 2024-09-13T00:29:34.030225+0000 mon.c (mon.2) 38 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:29:35.166 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:35 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.166 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:29:35 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.166 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:29:35 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.167 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:35 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.167 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:35 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.167 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:29:35 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.168 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:29:35 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.419 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:29:35 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.419 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:29:35 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.419 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:35 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.420 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:29:35 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.420 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:29:35 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.420 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:35 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.420 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[22596]: cluster 2024-09-13T00:29:33.749740+0000 mgr.y (mgr.14397) 10 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:35.420 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[22596]: cephadm 2024-09-13T00:29:34.016518+0000 mgr.y (mgr.14397) 11 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 4060M 2024-09-13T00:29:35.421 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[22596]: cephadm 2024-09-13T00:29:34.032588+0000 mgr.y (mgr.14397) 12 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.conf 2024-09-13T00:29:35.421 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[22596]: cephadm 2024-09-13T00:29:34.033099+0000 mgr.y (mgr.14397) 13 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:29:35.421 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:35 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.676 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:35 smithi088 bash[24895]: cluster 2024-09-13T00:29:33.749740+0000 mgr.y (mgr.14397) 10 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:35.676 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:35 smithi088 bash[24895]: cephadm 2024-09-13T00:29:34.016518+0000 mgr.y (mgr.14397) 11 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 4060M 2024-09-13T00:29:35.677 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:35 smithi088 bash[24895]: cephadm 2024-09-13T00:29:34.032588+0000 mgr.y (mgr.14397) 12 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.conf 2024-09-13T00:29:35.677 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:35 smithi088 bash[24895]: cephadm 2024-09-13T00:29:34.033099+0000 mgr.y (mgr.14397) 13 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:29:35.677 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:35 smithi088 bash[24895]: cephadm 2024-09-13T00:29:34.203208+0000 mgr.y (mgr.14397) 14 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:29:35.677 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:35 smithi088 bash[24895]: cephadm 2024-09-13T00:29:34.300806+0000 mgr.y (mgr.14397) 15 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:29:35.677 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:35 smithi088 bash[24895]: cephadm 2024-09-13T00:29:34.304690+0000 mgr.y (mgr.14397) 16 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:29:35.677 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:35 smithi088 bash[24895]: cephadm 2024-09-13T00:29:34.416820+0000 mgr.y (mgr.14397) 17 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:29:35.677 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:35 smithi088 bash[24895]: cephadm 2024-09-13T00:29:34.436479+0000 mgr.y (mgr.14397) 18 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:29:35.677 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:35 smithi088 bash[24895]: audit 2024-09-13T00:29:34.533278+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:35.677 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:35 smithi088 bash[24895]: audit 2024-09-13T00:29:34.540500+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:35.677 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:35 smithi088 bash[24895]: audit 2024-09-13T00:29:34.675094+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:35.678 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:35 smithi088 bash[24895]: audit 2024-09-13T00:29:34.688695+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:35.678 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:35 smithi088 bash[24895]: audit 2024-09-13T00:29:34.700668+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:35.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[22596]: cephadm 2024-09-13T00:29:34.203208+0000 mgr.y (mgr.14397) 14 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:29:35.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[22596]: cephadm 2024-09-13T00:29:34.300806+0000 mgr.y (mgr.14397) 15 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:29:35.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[22596]: cephadm 2024-09-13T00:29:34.304690+0000 mgr.y (mgr.14397) 16 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:29:35.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[22596]: cephadm 2024-09-13T00:29:34.416820+0000 mgr.y (mgr.14397) 17 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:29:35.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[22596]: cephadm 2024-09-13T00:29:34.436479+0000 mgr.y (mgr.14397) 18 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:29:35.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[22596]: audit 2024-09-13T00:29:34.533278+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:35.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[22596]: audit 2024-09-13T00:29:34.540500+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:35.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[22596]: audit 2024-09-13T00:29:34.675094+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:35.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[22596]: audit 2024-09-13T00:29:34.688695+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:35.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[22596]: audit 2024-09-13T00:29:34.700668+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:35.779 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:35 smithi077 systemd[1]: Started Ceph node-exporter.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:29:35.779 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[56099]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2024-09-13T00:29:35.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[32040]: cluster 2024-09-13T00:29:33.749740+0000 mgr.y (mgr.14397) 10 : cluster [DBG] pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:35.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[32040]: cephadm 2024-09-13T00:29:34.016518+0000 mgr.y (mgr.14397) 11 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 4060M 2024-09-13T00:29:35.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[32040]: cephadm 2024-09-13T00:29:34.032588+0000 mgr.y (mgr.14397) 12 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.conf 2024-09-13T00:29:35.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[32040]: cephadm 2024-09-13T00:29:34.033099+0000 mgr.y (mgr.14397) 13 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:29:35.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[32040]: cephadm 2024-09-13T00:29:34.203208+0000 mgr.y (mgr.14397) 14 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:29:35.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[32040]: cephadm 2024-09-13T00:29:34.300806+0000 mgr.y (mgr.14397) 15 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:29:35.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[32040]: cephadm 2024-09-13T00:29:34.304690+0000 mgr.y (mgr.14397) 16 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:29:35.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[32040]: cephadm 2024-09-13T00:29:34.416820+0000 mgr.y (mgr.14397) 17 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:29:35.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[32040]: cephadm 2024-09-13T00:29:34.436479+0000 mgr.y (mgr.14397) 18 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:29:35.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[32040]: audit 2024-09-13T00:29:34.533278+0000 mon.a (mon.0) 649 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:35.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[32040]: audit 2024-09-13T00:29:34.540500+0000 mon.a (mon.0) 650 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:35.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[32040]: audit 2024-09-13T00:29:34.675094+0000 mon.a (mon.0) 651 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:35.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[32040]: audit 2024-09-13T00:29:34.688695+0000 mon.a (mon.0) 652 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:35.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:35 smithi077 bash[32040]: audit 2024-09-13T00:29:34.700668+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:35.926 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.927 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.927 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 00:29:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.927 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.928 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:29:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.928 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:29:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.929 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:29:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.929 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:35.929 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:36.193 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:36 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:36.194 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:29:36 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:36.194 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:29:36 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:36.195 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 00:29:36 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:36.195 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:29:36 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:36.195 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:36 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:36.196 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:29:36 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:36.196 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:36 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:36.196 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:36 smithi088 systemd[1]: Started Ceph node-exporter.b for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:29:36.277 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[56099]: v1.5.0: Pulling from prometheus/node-exporter 2024-09-13T00:29:36.277 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[56099]: 22b70bddd3ac: Pulling fs layer 2024-09-13T00:29:36.277 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[56099]: 5c12815fee55: Pulling fs layer 2024-09-13T00:29:36.277 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[56099]: c0e87333d380: Pulling fs layer 2024-09-13T00:29:36.469 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:36 smithi088 bash[50193]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2024-09-13T00:29:36.777 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[56099]: 5c12815fee55: Verifying Checksum 2024-09-13T00:29:36.777 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[56099]: 5c12815fee55: Download complete 2024-09-13T00:29:36.777 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[56099]: 22b70bddd3ac: Verifying Checksum 2024-09-13T00:29:36.777 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[56099]: 22b70bddd3ac: Download complete 2024-09-13T00:29:36.778 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[56099]: c0e87333d380: Verifying Checksum 2024-09-13T00:29:36.778 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[56099]: c0e87333d380: Download complete 2024-09-13T00:29:36.778 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[56099]: 22b70bddd3ac: Pull complete 2024-09-13T00:29:36.778 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:29:36 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[22596]: cephadm 2024-09-13T00:29:34.564565+0000 mgr.y (mgr.14397) 19 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:29:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[22596]: cephadm 2024-09-13T00:29:34.708226+0000 mgr.y (mgr.14397) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi077 2024-09-13T00:29:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[22596]: audit 2024-09-13T00:29:35.470605+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[22596]: audit 2024-09-13T00:29:35.476414+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[22596]: audit 2024-09-13T00:29:35.482301+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[22596]: audit 2024-09-13T00:29:36.202602+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[22596]: audit 2024-09-13T00:29:36.214092+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[22596]: audit 2024-09-13T00:29:36.223475+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[22596]: audit 2024-09-13T00:29:36.231773+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[22596]: audit 2024-09-13T00:29:36.249346+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[32040]: cephadm 2024-09-13T00:29:34.564565+0000 mgr.y (mgr.14397) 19 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:29:36.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[32040]: cephadm 2024-09-13T00:29:34.708226+0000 mgr.y (mgr.14397) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi077 2024-09-13T00:29:36.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[32040]: audit 2024-09-13T00:29:35.470605+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[32040]: audit 2024-09-13T00:29:35.476414+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[32040]: audit 2024-09-13T00:29:35.482301+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[32040]: audit 2024-09-13T00:29:36.202602+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[32040]: audit 2024-09-13T00:29:36.214092+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[32040]: audit 2024-09-13T00:29:36.223475+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[32040]: audit 2024-09-13T00:29:36.231773+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:36 smithi077 bash[32040]: audit 2024-09-13T00:29:36.249346+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:36 smithi088 bash[24895]: cephadm 2024-09-13T00:29:34.564565+0000 mgr.y (mgr.14397) 19 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:29:36.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:36 smithi088 bash[24895]: cephadm 2024-09-13T00:29:34.708226+0000 mgr.y (mgr.14397) 20 : cephadm [INF] Deploying daemon node-exporter.a on smithi077 2024-09-13T00:29:36.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:36 smithi088 bash[24895]: audit 2024-09-13T00:29:35.470605+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:36 smithi088 bash[24895]: audit 2024-09-13T00:29:35.476414+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:36 smithi088 bash[24895]: audit 2024-09-13T00:29:35.482301+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:36 smithi088 bash[24895]: audit 2024-09-13T00:29:36.202602+0000 mon.a (mon.0) 657 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:36 smithi088 bash[24895]: audit 2024-09-13T00:29:36.214092+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:36 smithi088 bash[24895]: audit 2024-09-13T00:29:36.223475+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:36 smithi088 bash[24895]: audit 2024-09-13T00:29:36.231773+0000 mon.a (mon.0) 660 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:36.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:36 smithi088 bash[24895]: audit 2024-09-13T00:29:36.249346+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:37.286 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:36 smithi088 bash[50193]: v1.5.0: Pulling from prometheus/node-exporter 2024-09-13T00:29:37.286 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:37 smithi088 bash[50193]: 22b70bddd3ac: Pulling fs layer 2024-09-13T00:29:37.286 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:37 smithi088 bash[50193]: 5c12815fee55: Pulling fs layer 2024-09-13T00:29:37.286 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:37 smithi088 bash[50193]: c0e87333d380: Pulling fs layer 2024-09-13T00:29:37.467 INFO:teuthology.orchestra.run.smithi088.stdout:Scheduled grafana update... 2024-09-13T00:29:37.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:37 smithi088 bash[24895]: cephadm 2024-09-13T00:29:35.483188+0000 mgr.y (mgr.14397) 21 : cephadm [INF] Deploying daemon node-exporter.b on smithi088 2024-09-13T00:29:37.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:37 smithi088 bash[24895]: cluster 2024-09-13T00:29:35.750351+0000 mgr.y (mgr.14397) 22 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T00:29:37.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:37 smithi088 bash[24895]: cephadm 2024-09-13T00:29:36.274100+0000 mgr.y (mgr.14397) 23 : cephadm [INF] Deploying daemon alertmanager.a on smithi077 2024-09-13T00:29:37.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:37 smithi088 bash[24895]: audit 2024-09-13T00:29:37.470233+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:37.600 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:37 smithi088 bash[50193]: 22b70bddd3ac: Download complete 2024-09-13T00:29:37.601 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:37 smithi088 bash[50193]: 5c12815fee55: Verifying Checksum 2024-09-13T00:29:37.601 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:37 smithi088 bash[50193]: 5c12815fee55: Download complete 2024-09-13T00:29:37.601 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:37 smithi088 bash[50193]: 22b70bddd3ac: Pull complete 2024-09-13T00:29:37.601 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:37 smithi088 bash[50193]: c0e87333d380: Verifying Checksum 2024-09-13T00:29:37.601 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:37 smithi088 bash[50193]: c0e87333d380: Download complete 2024-09-13T00:29:37.777 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:37 smithi077 bash[56099]: 5c12815fee55: Pull complete 2024-09-13T00:29:37.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:37 smithi077 bash[32040]: cephadm 2024-09-13T00:29:35.483188+0000 mgr.y (mgr.14397) 21 : cephadm [INF] Deploying daemon node-exporter.b on smithi088 2024-09-13T00:29:37.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:37 smithi077 bash[32040]: cluster 2024-09-13T00:29:35.750351+0000 mgr.y (mgr.14397) 22 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T00:29:37.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:37 smithi077 bash[32040]: cephadm 2024-09-13T00:29:36.274100+0000 mgr.y (mgr.14397) 23 : cephadm [INF] Deploying daemon alertmanager.a on smithi077 2024-09-13T00:29:37.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:37 smithi077 bash[32040]: audit 2024-09-13T00:29:37.470233+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:37.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:37 smithi077 bash[22596]: cephadm 2024-09-13T00:29:35.483188+0000 mgr.y (mgr.14397) 21 : cephadm [INF] Deploying daemon node-exporter.b on smithi088 2024-09-13T00:29:37.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:37 smithi077 bash[22596]: cluster 2024-09-13T00:29:35.750351+0000 mgr.y (mgr.14397) 22 : cluster [DBG] pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 10 op/s 2024-09-13T00:29:37.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:37 smithi077 bash[22596]: cephadm 2024-09-13T00:29:36.274100+0000 mgr.y (mgr.14397) 23 : cephadm [INF] Deploying daemon alertmanager.a on smithi077 2024-09-13T00:29:37.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:37 smithi077 bash[22596]: audit 2024-09-13T00:29:37.470233+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:38.276 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:37 smithi077 bash[56099]: c0e87333d380: Pull complete 2024-09-13T00:29:38.457 DEBUG:teuthology.orchestra.run.smithi088:grafana.a> sudo journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@grafana.a.service 2024-09-13T00:29:38.459 INFO:tasks.cephadm:Setting up client nodes... 2024-09-13T00:29:38.459 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T00:29:38.527 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:38 smithi077 bash[56099]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2024-09-13T00:29:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:38 smithi077 bash[22596]: audit 2024-09-13T00:29:37.462533+0000 mgr.y (mgr.14397) 24 : audit [DBG] from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi088=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:29:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:38 smithi077 bash[22596]: cephadm 2024-09-13T00:29:37.464637+0000 mgr.y (mgr.14397) 25 : cephadm [INF] Saving service grafana spec with placement smithi088=a;count:1 2024-09-13T00:29:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:38 smithi077 bash[22596]: audit 2024-09-13T00:29:37.815209+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:38.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:38 smithi077 bash[32040]: audit 2024-09-13T00:29:37.462533+0000 mgr.y (mgr.14397) 24 : audit [DBG] from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi088=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:29:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:38 smithi077 bash[32040]: cephadm 2024-09-13T00:29:37.464637+0000 mgr.y (mgr.14397) 25 : cephadm [INF] Saving service grafana spec with placement smithi088=a;count:1 2024-09-13T00:29:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:38 smithi077 bash[32040]: audit 2024-09-13T00:29:37.815209+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:38.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:38 smithi088 bash[24895]: audit 2024-09-13T00:29:37.462533+0000 mgr.y (mgr.14397) 24 : audit [DBG] from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi088=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:29:38.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:38 smithi088 bash[24895]: cephadm 2024-09-13T00:29:37.464637+0000 mgr.y (mgr.14397) 25 : cephadm [INF] Saving service grafana spec with placement smithi088=a;count:1 2024-09-13T00:29:38.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:38 smithi088 bash[24895]: audit 2024-09-13T00:29:37.815209+0000 mon.a (mon.0) 663 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:38.601 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:38 smithi088 bash[50193]: 5c12815fee55: Pull complete 2024-09-13T00:29:38.777 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:38 smithi077 bash[56099]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2024-09-13T00:29:39.100 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:38 smithi088 bash[50193]: c0e87333d380: Pull complete 2024-09-13T00:29:39.101 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:38 smithi088 bash[50193]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2024-09-13T00:29:39.101 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:39 smithi088 bash[50193]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2024-09-13T00:29:39.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:39 smithi077 bash[22596]: cluster 2024-09-13T00:29:37.750753+0000 mgr.y (mgr.14397) 26 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T00:29:39.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:39 smithi077 bash[32040]: cluster 2024-09-13T00:29:37.750753+0000 mgr.y (mgr.14397) 26 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T00:29:39.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:39 smithi088 bash[24895]: cluster 2024-09-13T00:29:37.750753+0000 mgr.y (mgr.14397) 26 : cluster [DBG] pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 20 KiB/s rd, 0 B/s wr, 8 op/s 2024-09-13T00:29:40.489 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.267Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T00:29:40.489 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.267Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T00:29:40.489 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.268Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T00:29:40.489 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.268Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T00:29:40.489 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.268Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T00:29:40.489 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.268Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T00:29:40.489 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T00:29:40.489 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T00:29:40.489 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T00:29:40.489 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T00:29:40.490 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T00:29:40.490 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T00:29:40.490 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T00:29:40.490 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T00:29:40.490 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T00:29:40.490 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T00:29:40.490 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T00:29:40.490 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T00:29:40.490 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T00:29:40.490 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T00:29:40.490 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T00:29:40.490 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T00:29:40.491 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T00:29:40.491 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T00:29:40.491 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T00:29:40.491 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T00:29:40.491 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T00:29:40.491 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T00:29:40.491 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T00:29:40.491 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T00:29:40.491 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T00:29:40.492 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T00:29:40.492 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T00:29:40.492 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T00:29:40.492 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T00:29:40.492 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T00:29:40.492 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T00:29:40.492 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T00:29:40.492 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T00:29:40.492 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T00:29:40.492 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T00:29:40.492 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T00:29:40.493 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T00:29:40.493 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T00:29:40.493 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T00:29:40.493 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T00:29:40.493 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T00:29:40.493 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T00:29:40.493 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T00:29:40.493 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T00:29:40.493 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T00:29:40.493 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T00:29:40.493 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[50193]: ts=2024-09-13T00:29:40.269Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T00:29:40.759 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[24895]: cluster 2024-09-13T00:29:39.751261+0000 mgr.y (mgr.14397) 27 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:29:40.863 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:40 smithi077 bash[22596]: cluster 2024-09-13T00:29:39.751261+0000 mgr.y (mgr.14397) 27 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:29:40.863 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:40 smithi077 bash[32040]: cluster 2024-09-13T00:29:39.751261+0000 mgr.y (mgr.14397) 27 : cluster [DBG] pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 16 KiB/s rd, 0 B/s wr, 6 op/s 2024-09-13T00:29:41.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:40 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:29:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:29:42.028 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.664Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T00:29:42.028 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.664Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T00:29:42.028 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.665Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T00:29:42.028 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.665Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T00:29:42.028 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T00:29:42.028 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T00:29:42.028 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T00:29:42.028 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T00:29:42.028 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T00:29:42.029 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T00:29:42.029 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T00:29:42.029 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T00:29:42.029 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T00:29:42.029 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T00:29:42.029 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T00:29:42.029 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T00:29:42.029 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T00:29:42.029 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T00:29:42.029 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T00:29:42.029 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T00:29:42.029 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T00:29:42.029 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T00:29:42.030 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T00:29:42.030 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T00:29:42.030 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T00:29:42.030 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T00:29:42.030 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T00:29:42.030 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T00:29:42.030 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T00:29:42.031 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T00:29:42.031 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T00:29:42.031 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T00:29:42.031 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T00:29:42.031 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T00:29:42.031 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T00:29:42.031 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T00:29:42.031 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T00:29:42.031 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T00:29:42.031 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T00:29:42.031 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T00:29:42.031 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T00:29:42.032 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T00:29:42.032 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T00:29:42.032 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T00:29:42.032 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T00:29:42.032 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T00:29:42.032 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T00:29:42.032 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T00:29:42.032 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T00:29:42.032 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T00:29:42.032 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.666Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T00:29:42.032 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.667Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T00:29:42.032 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:41 smithi077 bash[56099]: ts=2024-09-13T00:29:41.667Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T00:29:43.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:42 smithi088 bash[24895]: cluster 2024-09-13T00:29:41.751628+0000 mgr.y (mgr.14397) 28 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:29:43.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:42 smithi077 bash[22596]: cluster 2024-09-13T00:29:41.751628+0000 mgr.y (mgr.14397) 28 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:29:43.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:42 smithi077 bash[32040]: cluster 2024-09-13T00:29:41.751628+0000 mgr.y (mgr.14397) 28 : cluster [DBG] pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:29:44.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:43 smithi077 bash[22596]: audit 2024-09-13T00:29:42.819556+0000 mon.c (mon.2) 39 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:44.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:43 smithi077 bash[32040]: audit 2024-09-13T00:29:42.819556+0000 mon.c (mon.2) 39 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:44.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:43 smithi088 bash[24895]: audit 2024-09-13T00:29:42.819556+0000 mon.c (mon.2) 39 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:44.214 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:29:45.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:44 smithi088 bash[24895]: cluster 2024-09-13T00:29:43.752280+0000 mgr.y (mgr.14397) 29 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:29:45.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:44 smithi077 bash[22596]: cluster 2024-09-13T00:29:43.752280+0000 mgr.y (mgr.14397) 29 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:29:45.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:44 smithi077 bash[32040]: cluster 2024-09-13T00:29:43.752280+0000 mgr.y (mgr.14397) 29 : cluster [DBG] pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:29:47.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:46 smithi088 bash[24895]: cluster 2024-09-13T00:29:45.753096+0000 mgr.y (mgr.14397) 30 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:29:47.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:46 smithi077 bash[22596]: cluster 2024-09-13T00:29:45.753096+0000 mgr.y (mgr.14397) 30 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:29:47.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:46 smithi077 bash[32040]: cluster 2024-09-13T00:29:45.753096+0000 mgr.y (mgr.14397) 30 : cluster [DBG] pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:29:48.822 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:48 smithi077 bash[22596]: cluster 2024-09-13T00:29:47.753717+0000 mgr.y (mgr.14397) 31 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:49.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:48 smithi088 bash[24895]: cluster 2024-09-13T00:29:47.753717+0000 mgr.y (mgr.14397) 31 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:49.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:48 smithi077 bash[32040]: cluster 2024-09-13T00:29:47.753717+0000 mgr.y (mgr.14397) 31 : cluster [DBG] pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:50.010 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.011 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:29:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.011 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:29:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.011 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:29:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.011 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:29:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.012 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:29:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.012 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:29:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.012 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.012 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.013 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.013 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:29:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.277 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:29:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.277 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:29:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.277 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:29:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.278 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:29:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.278 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:29:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.278 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:29:50 smithi077 systemd[1]: Started Ceph alertmanager.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:29:50.278 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.279 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:29:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:29:50.399 INFO:teuthology.orchestra.run.smithi077.stdout:[client.0] 2024-09-13T00:29:50.399 INFO:teuthology.orchestra.run.smithi077.stdout: key = AQB+h+NmlynYFxAAB4RkCzGCTvP3VE2zYADoFg== 2024-09-13T00:29:51.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:29:50 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:29:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:29:51.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[22596]: cluster 2024-09-13T00:29:49.754402+0000 mgr.y (mgr.14397) 32 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:51.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[22596]: audit 2024-09-13T00:29:50.306572+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[22596]: audit 2024-09-13T00:29:50.317421+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[22596]: audit 2024-09-13T00:29:50.322468+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[22596]: audit 2024-09-13T00:29:50.327232+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[22596]: audit 2024-09-13T00:29:50.356864+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:29:51.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[22596]: audit 2024-09-13T00:29:50.399898+0000 mon.a (mon.0) 668 : audit [INF] from='client.? 172.21.15.77:0/1199127911' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T00:29:51.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[22596]: audit 2024-09-13T00:29:50.403615+0000 mon.a (mon.0) 669 : audit [INF] from='client.? 172.21.15.77:0/1199127911' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T00:29:51.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[22596]: audit 2024-09-13T00:29:50.591745+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[22596]: audit 2024-09-13T00:29:50.596248+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[32040]: cluster 2024-09-13T00:29:49.754402+0000 mgr.y (mgr.14397) 32 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:51.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[32040]: audit 2024-09-13T00:29:50.306572+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[32040]: audit 2024-09-13T00:29:50.317421+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[32040]: audit 2024-09-13T00:29:50.322468+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[32040]: audit 2024-09-13T00:29:50.327232+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[32040]: audit 2024-09-13T00:29:50.356864+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:29:51.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[32040]: audit 2024-09-13T00:29:50.399898+0000 mon.a (mon.0) 668 : audit [INF] from='client.? 172.21.15.77:0/1199127911' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T00:29:51.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[32040]: audit 2024-09-13T00:29:50.403615+0000 mon.a (mon.0) 669 : audit [INF] from='client.? 172.21.15.77:0/1199127911' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T00:29:51.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[32040]: audit 2024-09-13T00:29:50.591745+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:51 smithi077 bash[32040]: audit 2024-09-13T00:29:50.596248+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:51 smithi088 bash[24895]: cluster 2024-09-13T00:29:49.754402+0000 mgr.y (mgr.14397) 32 : cluster [DBG] pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:51 smithi088 bash[24895]: audit 2024-09-13T00:29:50.306572+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:51 smithi088 bash[24895]: audit 2024-09-13T00:29:50.317421+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:51 smithi088 bash[24895]: audit 2024-09-13T00:29:50.322468+0000 mon.a (mon.0) 666 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:51 smithi088 bash[24895]: audit 2024-09-13T00:29:50.327232+0000 mon.a (mon.0) 667 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:51 smithi088 bash[24895]: audit 2024-09-13T00:29:50.356864+0000 mon.c (mon.2) 40 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:29:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:51 smithi088 bash[24895]: audit 2024-09-13T00:29:50.399898+0000 mon.a (mon.0) 668 : audit [INF] from='client.? 172.21.15.77:0/1199127911' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T00:29:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:51 smithi088 bash[24895]: audit 2024-09-13T00:29:50.403615+0000 mon.a (mon.0) 669 : audit [INF] from='client.? 172.21.15.77:0/1199127911' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T00:29:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:51 smithi088 bash[24895]: audit 2024-09-13T00:29:50.591745+0000 mon.a (mon.0) 670 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:51 smithi088 bash[24895]: audit 2024-09-13T00:29:50.596248+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:51.796 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:29:51.796 DEBUG:teuthology.orchestra.run.smithi077:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-13T00:29:51.796 DEBUG:teuthology.orchestra.run.smithi077:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-13T00:29:51.817 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-13T00:29:52.277 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:29:52 smithi077 bash[56627]: ts=2024-09-13T00:29:52.099Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-13T00:29:52.277 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:29:52 smithi077 bash[56627]: ts=2024-09-13T00:29:52.099Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-13T00:29:52.277 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:29:52 smithi077 bash[56627]: ts=2024-09-13T00:29:52.100Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.77 port=9094 2024-09-13T00:29:52.278 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:29:52 smithi077 bash[56627]: ts=2024-09-13T00:29:52.101Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T00:29:52.278 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:29:52 smithi077 bash[56627]: ts=2024-09-13T00:29:52.131Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T00:29:52.278 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:29:52 smithi077 bash[56627]: ts=2024-09-13T00:29:52.131Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T00:29:52.278 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:29:52 smithi077 bash[56627]: ts=2024-09-13T00:29:52.134Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-13T00:29:52.278 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:29:52 smithi077 bash[56627]: ts=2024-09-13T00:29:52.134Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T00:29:53.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:53 smithi088 bash[24895]: cluster 2024-09-13T00:29:51.755150+0000 mgr.y (mgr.14397) 33 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:53.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:53 smithi088 bash[24895]: audit 2024-09-13T00:29:52.842610+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:53.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:53 smithi077 bash[22596]: cluster 2024-09-13T00:29:51.755150+0000 mgr.y (mgr.14397) 33 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:53.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:53 smithi077 bash[22596]: audit 2024-09-13T00:29:52.842610+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:53.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:53 smithi077 bash[32040]: cluster 2024-09-13T00:29:51.755150+0000 mgr.y (mgr.14397) 33 : cluster [DBG] pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:53.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:53 smithi077 bash[32040]: audit 2024-09-13T00:29:52.842610+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:54.527 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:29:54 smithi077 bash[56627]: ts=2024-09-13T00:29:54.101Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000357967s 2024-09-13T00:29:55.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:55 smithi088 bash[24895]: cluster 2024-09-13T00:29:53.755766+0000 mgr.y (mgr.14397) 34 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:55.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:55 smithi077 bash[32040]: cluster 2024-09-13T00:29:53.755766+0000 mgr.y (mgr.14397) 34 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:55.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:55 smithi077 bash[22596]: cluster 2024-09-13T00:29:53.755766+0000 mgr.y (mgr.14397) 34 : cluster [DBG] pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:56.146 INFO:teuthology.orchestra.run.smithi088.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.b/config 2024-09-13T00:29:56.950 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:56 smithi077 bash[22596]: audit 2024-09-13T00:29:55.689866+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:56.950 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:56 smithi077 bash[22596]: audit 2024-09-13T00:29:55.704377+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:56.950 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:56 smithi077 bash[22596]: cluster 2024-09-13T00:29:55.756385+0000 mgr.y (mgr.14397) 35 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:56.950 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:56 smithi077 bash[32040]: audit 2024-09-13T00:29:55.689866+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:56.950 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:56 smithi077 bash[32040]: audit 2024-09-13T00:29:55.704377+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:56.950 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:56 smithi077 bash[32040]: cluster 2024-09-13T00:29:55.756385+0000 mgr.y (mgr.14397) 35 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:57.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:56 smithi088 bash[24895]: audit 2024-09-13T00:29:55.689866+0000 mon.a (mon.0) 673 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:57.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:56 smithi088 bash[24895]: audit 2024-09-13T00:29:55.704377+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:57.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:56 smithi088 bash[24895]: cluster 2024-09-13T00:29:55.756385+0000 mgr.y (mgr.14397) 35 : cluster [DBG] pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:57.277 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:56 smithi077 bash[22845]: [13/Sep/2024:00:29:56] ENGINE Bus STOPPING 2024-09-13T00:29:57.277 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:56 smithi077 bash[22845]: [13/Sep/2024:00:29:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:29:57.277 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:56 smithi077 bash[22845]: [13/Sep/2024:00:29:56] ENGINE Bus STOPPED 2024-09-13T00:29:57.277 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[22845]: [13/Sep/2024:00:29:57] ENGINE Bus STARTING 2024-09-13T00:29:57.277 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[22845]: [13/Sep/2024:00:29:57] ENGINE Serving on http://:::9283 2024-09-13T00:29:57.277 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[22845]: [13/Sep/2024:00:29:57] ENGINE Bus STARTED 2024-09-13T00:29:58.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:57 smithi088 bash[24895]: audit 2024-09-13T00:29:56.848653+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:57 smithi088 bash[24895]: audit 2024-09-13T00:29:56.853926+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:57 smithi088 bash[24895]: audit 2024-09-13T00:29:56.856523+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:29:58.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:57 smithi088 bash[24895]: audit 2024-09-13T00:29:56.857231+0000 mon.c (mon.2) 42 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:29:58.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:57 smithi088 bash[24895]: audit 2024-09-13T00:29:56.861823+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:57 smithi088 bash[24895]: audit 2024-09-13T00:29:56.920639+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:57 smithi088 bash[24895]: audit 2024-09-13T00:29:56.927538+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:57 smithi088 bash[24895]: audit 2024-09-13T00:29:56.933438+0000 mon.c (mon.2) 43 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T00:29:58.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:57 smithi088 bash[24895]: audit 2024-09-13T00:29:56.934321+0000 mgr.y (mgr.14397) 36 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T00:29:58.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:57 smithi088 bash[24895]: audit 2024-09-13T00:29:56.943083+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:57 smithi088 bash[24895]: cephadm 2024-09-13T00:29:56.967307+0000 mgr.y (mgr.14397) 37 : cephadm [INF] Deploying daemon grafana.a on smithi088 2024-09-13T00:29:58.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:57 smithi088 bash[24895]: audit 2024-09-13T00:29:57.820308+0000 mon.c (mon.2) 44 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:58.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[22596]: audit 2024-09-13T00:29:56.848653+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[22596]: audit 2024-09-13T00:29:56.853926+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[22596]: audit 2024-09-13T00:29:56.856523+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:29:58.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[22596]: audit 2024-09-13T00:29:56.857231+0000 mon.c (mon.2) 42 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:29:58.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[22596]: audit 2024-09-13T00:29:56.861823+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[22596]: audit 2024-09-13T00:29:56.920639+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[22596]: audit 2024-09-13T00:29:56.927538+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[22596]: audit 2024-09-13T00:29:56.933438+0000 mon.c (mon.2) 43 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T00:29:58.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[22596]: audit 2024-09-13T00:29:56.934321+0000 mgr.y (mgr.14397) 36 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T00:29:58.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[22596]: audit 2024-09-13T00:29:56.943083+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[22596]: cephadm 2024-09-13T00:29:56.967307+0000 mgr.y (mgr.14397) 37 : cephadm [INF] Deploying daemon grafana.a on smithi088 2024-09-13T00:29:58.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[22596]: audit 2024-09-13T00:29:57.820308+0000 mon.c (mon.2) 44 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:58.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[32040]: audit 2024-09-13T00:29:56.848653+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[32040]: audit 2024-09-13T00:29:56.853926+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[32040]: audit 2024-09-13T00:29:56.856523+0000 mon.c (mon.2) 41 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:29:58.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[32040]: audit 2024-09-13T00:29:56.857231+0000 mon.c (mon.2) 42 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:29:58.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[32040]: audit 2024-09-13T00:29:56.861823+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[32040]: audit 2024-09-13T00:29:56.920639+0000 mon.a (mon.0) 678 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[32040]: audit 2024-09-13T00:29:56.927538+0000 mon.a (mon.0) 679 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[32040]: audit 2024-09-13T00:29:56.933438+0000 mon.c (mon.2) 43 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T00:29:58.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[32040]: audit 2024-09-13T00:29:56.934321+0000 mgr.y (mgr.14397) 36 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-13T00:29:58.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[32040]: audit 2024-09-13T00:29:56.943083+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:29:58.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[32040]: cephadm 2024-09-13T00:29:56.967307+0000 mgr.y (mgr.14397) 37 : cephadm [INF] Deploying daemon grafana.a on smithi088 2024-09-13T00:29:58.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:57 smithi077 bash[32040]: audit 2024-09-13T00:29:57.820308+0000 mon.c (mon.2) 44 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:29:59.199 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:58 smithi088 bash[24895]: cluster 2024-09-13T00:29:57.756941+0000 mgr.y (mgr.14397) 38 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:59.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:58 smithi077 bash[22596]: cluster 2024-09-13T00:29:57.756941+0000 mgr.y (mgr.14397) 38 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:59.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:58 smithi077 bash[32040]: cluster 2024-09-13T00:29:57.756941+0000 mgr.y (mgr.14397) 38 : cluster [DBG] pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:29:59.584 INFO:teuthology.orchestra.run.smithi088.stdout:[client.1] 2024-09-13T00:29:59.585 INFO:teuthology.orchestra.run.smithi088.stdout: key = AQCHh+NmH9WrIhAA9SXQao3tVqlmA/DfmGp+0Q== 2024-09-13T00:30:00.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:59 smithi077 bash[22596]: audit 2024-09-13T00:29:59.580297+0000 mon.b (mon.1) 29 : audit [INF] from='client.? 172.21.15.88:0/3053039724' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T00:30:00.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:59 smithi077 bash[22596]: audit 2024-09-13T00:29:59.581364+0000 mon.a (mon.0) 681 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T00:30:00.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:29:59 smithi077 bash[22596]: audit 2024-09-13T00:29:59.588830+0000 mon.a (mon.0) 682 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T00:30:00.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:59 smithi077 bash[32040]: audit 2024-09-13T00:29:59.580297+0000 mon.b (mon.1) 29 : audit [INF] from='client.? 172.21.15.88:0/3053039724' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T00:30:00.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:59 smithi077 bash[32040]: audit 2024-09-13T00:29:59.581364+0000 mon.a (mon.0) 681 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T00:30:00.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:29:59 smithi077 bash[32040]: audit 2024-09-13T00:29:59.588830+0000 mon.a (mon.0) 682 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T00:30:00.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:59 smithi088 bash[24895]: audit 2024-09-13T00:29:59.580297+0000 mon.b (mon.1) 29 : audit [INF] from='client.? 172.21.15.88:0/3053039724' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T00:30:00.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:59 smithi088 bash[24895]: audit 2024-09-13T00:29:59.581364+0000 mon.a (mon.0) 681 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-13T00:30:00.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:29:59 smithi088 bash[24895]: audit 2024-09-13T00:29:59.588830+0000 mon.a (mon.0) 682 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-13T00:30:00.878 DEBUG:teuthology.orchestra.run.smithi088:> set -ex 2024-09-13T00:30:00.879 DEBUG:teuthology.orchestra.run.smithi088:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-13T00:30:00.879 DEBUG:teuthology.orchestra.run.smithi088:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-13T00:30:00.901 INFO:tasks.cephadm:Creating RBD pool 2024-09-13T00:30:00.902 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd pool create rbd 8 2024-09-13T00:30:01.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:30:00 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:30:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:30:01.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:00 smithi088 bash[24895]: cluster 2024-09-13T00:29:59.757548+0000 mgr.y (mgr.14397) 39 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:01.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:00 smithi088 bash[24895]: cluster 2024-09-13T00:30:00.000150+0000 mon.a (mon.0) 683 : cluster [INF] overall HEALTH_OK 2024-09-13T00:30:01.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:00 smithi077 bash[22596]: cluster 2024-09-13T00:29:59.757548+0000 mgr.y (mgr.14397) 39 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:01.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:00 smithi077 bash[22596]: cluster 2024-09-13T00:30:00.000150+0000 mon.a (mon.0) 683 : cluster [INF] overall HEALTH_OK 2024-09-13T00:30:01.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:00 smithi077 bash[32040]: cluster 2024-09-13T00:29:59.757548+0000 mgr.y (mgr.14397) 39 : cluster [DBG] pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:01.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:00 smithi077 bash[32040]: cluster 2024-09-13T00:30:00.000150+0000 mon.a (mon.0) 683 : cluster [INF] overall HEALTH_OK 2024-09-13T00:30:02.527 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:02 smithi077 bash[56627]: ts=2024-09-13T00:30:02.105Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003659889s 2024-09-13T00:30:03.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:02 smithi077 bash[22596]: cluster 2024-09-13T00:30:01.758110+0000 mgr.y (mgr.14397) 40 : cluster [DBG] pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:03.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:02 smithi077 bash[32040]: cluster 2024-09-13T00:30:01.758110+0000 mgr.y (mgr.14397) 40 : cluster [DBG] pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:03.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:02 smithi088 bash[24895]: cluster 2024-09-13T00:30:01.758110+0000 mgr.y (mgr.14397) 40 : cluster [DBG] pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:05.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:04 smithi077 bash[22596]: cluster 2024-09-13T00:30:03.758708+0000 mgr.y (mgr.14397) 41 : cluster [DBG] pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:05.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:04 smithi077 bash[32040]: cluster 2024-09-13T00:30:03.758708+0000 mgr.y (mgr.14397) 41 : cluster [DBG] pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:05.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:04 smithi088 bash[24895]: cluster 2024-09-13T00:30:03.758708+0000 mgr.y (mgr.14397) 41 : cluster [DBG] pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:05.683 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:30:07.220 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:06 smithi077 bash[22596]: cluster 2024-09-13T00:30:05.759329+0000 mgr.y (mgr.14397) 42 : cluster [DBG] pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:07.221 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:06 smithi077 bash[32040]: cluster 2024-09-13T00:30:05.759329+0000 mgr.y (mgr.14397) 42 : cluster [DBG] pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:07.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:06 smithi088 bash[24895]: cluster 2024-09-13T00:30:05.759329+0000 mgr.y (mgr.14397) 42 : cluster [DBG] pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:09.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:08 smithi077 bash[22596]: cluster 2024-09-13T00:30:07.759901+0000 mgr.y (mgr.14397) 43 : cluster [DBG] pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:09.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:08 smithi077 bash[32040]: cluster 2024-09-13T00:30:07.759901+0000 mgr.y (mgr.14397) 43 : cluster [DBG] pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:09.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:08 smithi088 bash[24895]: cluster 2024-09-13T00:30:07.759901+0000 mgr.y (mgr.14397) 43 : cluster [DBG] pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:09.983 INFO:teuthology.orchestra.run.smithi077.stderr:pool 'rbd' created 2024-09-13T00:30:10.221 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:09 smithi077 bash[22596]: audit 2024-09-13T00:30:09.450611+0000 mon.c (mon.2) 45 : audit [INF] from='client.? 172.21.15.77:0/4189169509' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T00:30:10.221 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:09 smithi077 bash[22596]: audit 2024-09-13T00:30:09.451106+0000 mon.a (mon.0) 684 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T00:30:10.221 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:09 smithi077 bash[32040]: audit 2024-09-13T00:30:09.450611+0000 mon.c (mon.2) 45 : audit [INF] from='client.? 172.21.15.77:0/4189169509' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T00:30:10.222 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:09 smithi077 bash[32040]: audit 2024-09-13T00:30:09.451106+0000 mon.a (mon.0) 684 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T00:30:10.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:09 smithi088 bash[24895]: audit 2024-09-13T00:30:09.450611+0000 mon.c (mon.2) 45 : audit [INF] from='client.? 172.21.15.77:0/4189169509' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T00:30:10.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:09 smithi088 bash[24895]: audit 2024-09-13T00:30:09.451106+0000 mon.a (mon.0) 684 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-13T00:30:10.819 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- rbd pool init rbd 2024-09-13T00:30:11.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:30:10 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:30:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:30:11.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:10 smithi088 bash[24895]: cluster 2024-09-13T00:30:09.760472+0000 mgr.y (mgr.14397) 44 : cluster [DBG] pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:11.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:10 smithi088 bash[24895]: audit 2024-09-13T00:30:09.984456+0000 mon.a (mon.0) 685 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T00:30:11.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:10 smithi088 bash[24895]: cluster 2024-09-13T00:30:09.984615+0000 mon.a (mon.0) 686 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2024-09-13T00:30:11.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:10 smithi077 bash[22596]: cluster 2024-09-13T00:30:09.760472+0000 mgr.y (mgr.14397) 44 : cluster [DBG] pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:11.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:10 smithi077 bash[22596]: audit 2024-09-13T00:30:09.984456+0000 mon.a (mon.0) 685 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T00:30:11.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:10 smithi077 bash[22596]: cluster 2024-09-13T00:30:09.984615+0000 mon.a (mon.0) 686 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2024-09-13T00:30:11.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:10 smithi077 bash[32040]: cluster 2024-09-13T00:30:09.760472+0000 mgr.y (mgr.14397) 44 : cluster [DBG] pgmap v24: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:11.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:10 smithi077 bash[32040]: audit 2024-09-13T00:30:09.984456+0000 mon.a (mon.0) 685 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-13T00:30:11.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:10 smithi077 bash[32040]: cluster 2024-09-13T00:30:09.984615+0000 mon.a (mon.0) 686 : cluster [DBG] osdmap e52: 8 total, 8 up, 8 in 2024-09-13T00:30:12.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:12 smithi077 bash[22596]: cluster 2024-09-13T00:30:10.995033+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2024-09-13T00:30:12.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:12 smithi077 bash[32040]: cluster 2024-09-13T00:30:10.995033+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2024-09-13T00:30:12.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:12 smithi088 bash[24895]: cluster 2024-09-13T00:30:10.995033+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e53: 8 total, 8 up, 8 in 2024-09-13T00:30:13.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:13 smithi077 bash[32040]: cluster 2024-09-13T00:30:11.761092+0000 mgr.y (mgr.14397) 45 : cluster [DBG] pgmap v27: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:13.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:13 smithi077 bash[32040]: cluster 2024-09-13T00:30:11.990254+0000 mon.a (mon.0) 688 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:30:13.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:13 smithi077 bash[32040]: cluster 2024-09-13T00:30:12.011656+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-09-13T00:30:13.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:13 smithi077 bash[32040]: audit 2024-09-13T00:30:12.828293+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:13.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:13 smithi077 bash[32040]: audit 2024-09-13T00:30:12.830707+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:13.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:13 smithi077 bash[22596]: cluster 2024-09-13T00:30:11.761092+0000 mgr.y (mgr.14397) 45 : cluster [DBG] pgmap v27: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:13.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:13 smithi077 bash[22596]: cluster 2024-09-13T00:30:11.990254+0000 mon.a (mon.0) 688 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:30:13.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:13 smithi077 bash[22596]: cluster 2024-09-13T00:30:12.011656+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-09-13T00:30:13.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:13 smithi077 bash[22596]: audit 2024-09-13T00:30:12.828293+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:13.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:13 smithi077 bash[22596]: audit 2024-09-13T00:30:12.830707+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:13.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:13 smithi088 bash[24895]: cluster 2024-09-13T00:30:11.761092+0000 mgr.y (mgr.14397) 45 : cluster [DBG] pgmap v27: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:13.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:13 smithi088 bash[24895]: cluster 2024-09-13T00:30:11.990254+0000 mon.a (mon.0) 688 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:30:13.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:13 smithi088 bash[24895]: cluster 2024-09-13T00:30:12.011656+0000 mon.a (mon.0) 689 : cluster [DBG] osdmap e54: 8 total, 8 up, 8 in 2024-09-13T00:30:13.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:13 smithi088 bash[24895]: audit 2024-09-13T00:30:12.828293+0000 mon.a (mon.0) 690 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:13.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:13 smithi088 bash[24895]: audit 2024-09-13T00:30:12.830707+0000 mon.c (mon.2) 46 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:15.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:15 smithi077 bash[22596]: cluster 2024-09-13T00:30:13.761717+0000 mgr.y (mgr.14397) 46 : cluster [DBG] pgmap v29: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:15.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:15 smithi077 bash[32040]: cluster 2024-09-13T00:30:13.761717+0000 mgr.y (mgr.14397) 46 : cluster [DBG] pgmap v29: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:15.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:15 smithi088 bash[24895]: cluster 2024-09-13T00:30:13.761717+0000 mgr.y (mgr.14397) 46 : cluster [DBG] pgmap v29: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:15.596 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:30:17.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:17 smithi077 bash[32040]: cluster 2024-09-13T00:30:15.762433+0000 mgr.y (mgr.14397) 47 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:17.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:17 smithi077 bash[22596]: cluster 2024-09-13T00:30:15.762433+0000 mgr.y (mgr.14397) 47 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:17.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:17 smithi088 bash[24895]: cluster 2024-09-13T00:30:15.762433+0000 mgr.y (mgr.14397) 47 : cluster [DBG] pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:19.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:19 smithi077 bash[22596]: cluster 2024-09-13T00:30:17.763041+0000 mgr.y (mgr.14397) 48 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:19.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:19 smithi077 bash[32040]: cluster 2024-09-13T00:30:17.763041+0000 mgr.y (mgr.14397) 48 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:19.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:19 smithi088 bash[24895]: cluster 2024-09-13T00:30:17.763041+0000 mgr.y (mgr.14397) 48 : cluster [DBG] pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:20.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:20 smithi077 bash[22596]: audit 2024-09-13T00:30:19.276294+0000 mon.c (mon.2) 47 : audit [INF] from='client.? 172.21.15.77:0/1238100889' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T00:30:20.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:20 smithi077 bash[22596]: audit 2024-09-13T00:30:19.276644+0000 mon.a (mon.0) 691 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T00:30:20.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:20 smithi077 bash[32040]: audit 2024-09-13T00:30:19.276294+0000 mon.c (mon.2) 47 : audit [INF] from='client.? 172.21.15.77:0/1238100889' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T00:30:20.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:20 smithi077 bash[32040]: audit 2024-09-13T00:30:19.276644+0000 mon.a (mon.0) 691 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T00:30:20.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:20 smithi088 bash[24895]: audit 2024-09-13T00:30:19.276294+0000 mon.c (mon.2) 47 : audit [INF] from='client.? 172.21.15.77:0/1238100889' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T00:30:20.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:20 smithi088 bash[24895]: audit 2024-09-13T00:30:19.276644+0000 mon.a (mon.0) 691 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-13T00:30:21.034 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:30:20 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:30:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:30:21.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:21 smithi088 bash[24895]: cluster 2024-09-13T00:30:19.763787+0000 mgr.y (mgr.14397) 49 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:21.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:21 smithi088 bash[24895]: audit 2024-09-13T00:30:20.031394+0000 mon.a (mon.0) 692 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T00:30:21.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:21 smithi088 bash[24895]: cluster 2024-09-13T00:30:20.031588+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-09-13T00:30:21.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:21 smithi077 bash[22596]: cluster 2024-09-13T00:30:19.763787+0000 mgr.y (mgr.14397) 49 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:21.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:21 smithi077 bash[22596]: audit 2024-09-13T00:30:20.031394+0000 mon.a (mon.0) 692 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T00:30:21.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:21 smithi077 bash[22596]: cluster 2024-09-13T00:30:20.031588+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-09-13T00:30:21.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:21 smithi077 bash[32040]: cluster 2024-09-13T00:30:19.763787+0000 mgr.y (mgr.14397) 49 : cluster [DBG] pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:21.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:21 smithi077 bash[32040]: audit 2024-09-13T00:30:20.031394+0000 mon.a (mon.0) 692 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-13T00:30:21.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:21 smithi077 bash[32040]: cluster 2024-09-13T00:30:20.031588+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e55: 8 total, 8 up, 8 in 2024-09-13T00:30:22.327 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:22 smithi077 bash[22596]: cluster 2024-09-13T00:30:21.034208+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-09-13T00:30:22.328 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:22 smithi077 bash[32040]: cluster 2024-09-13T00:30:21.034208+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-09-13T00:30:22.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:22 smithi088 bash[24895]: cluster 2024-09-13T00:30:21.034208+0000 mon.a (mon.0) 694 : cluster [DBG] osdmap e56: 8 total, 8 up, 8 in 2024-09-13T00:30:22.870 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T00:30:22.871 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T00:30:22.871 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T00:30:23.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:23 smithi077 bash[22596]: cluster 2024-09-13T00:30:21.764355+0000 mgr.y (mgr.14397) 50 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:23.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:23 smithi077 bash[22596]: cluster 2024-09-13T00:30:22.036365+0000 mon.a (mon.0) 695 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:30:23.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:23 smithi077 bash[22596]: cluster 2024-09-13T00:30:22.036417+0000 mon.a (mon.0) 696 : cluster [INF] Cluster is now healthy 2024-09-13T00:30:23.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:23 smithi077 bash[22596]: cluster 2024-09-13T00:30:22.051543+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-09-13T00:30:23.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:23 smithi077 bash[32040]: cluster 2024-09-13T00:30:21.764355+0000 mgr.y (mgr.14397) 50 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:23.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:23 smithi077 bash[32040]: cluster 2024-09-13T00:30:22.036365+0000 mon.a (mon.0) 695 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:30:23.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:23 smithi077 bash[32040]: cluster 2024-09-13T00:30:22.036417+0000 mon.a (mon.0) 696 : cluster [INF] Cluster is now healthy 2024-09-13T00:30:23.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:23 smithi077 bash[32040]: cluster 2024-09-13T00:30:22.051543+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-09-13T00:30:23.318 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:23 smithi088 bash[24895]: cluster 2024-09-13T00:30:21.764355+0000 mgr.y (mgr.14397) 50 : cluster [DBG] pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:23.318 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:23 smithi088 bash[24895]: cluster 2024-09-13T00:30:22.036365+0000 mon.a (mon.0) 695 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:30:23.318 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:23 smithi088 bash[24895]: cluster 2024-09-13T00:30:22.036417+0000 mon.a (mon.0) 696 : cluster [INF] Cluster is now healthy 2024-09-13T00:30:23.318 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:23 smithi088 bash[24895]: cluster 2024-09-13T00:30:22.051543+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e57: 8 total, 8 up, 8 in 2024-09-13T00:30:25.170 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:25 smithi088 bash[24895]: cluster 2024-09-13T00:30:23.765043+0000 mgr.y (mgr.14397) 51 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:25.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:25 smithi077 bash[32040]: cluster 2024-09-13T00:30:23.765043+0000 mgr.y (mgr.14397) 51 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:25.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:25 smithi077 bash[22596]: cluster 2024-09-13T00:30:23.765043+0000 mgr.y (mgr.14397) 51 : cluster [DBG] pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:25.786 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:25.787 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:25.787 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:25.788 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:25.788 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:25.788 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:25.789 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:25.789 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:25.789 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:26.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:26.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:26.101 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:26.101 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:26.101 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:26.101 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:26.102 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:26.102 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:26.102 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:26.102 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:26.102 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:26 smithi088 systemd[1]: Started Ceph grafana.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:30:26.103 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:30:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:30:27.310 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[24895]: cluster 2024-09-13T00:30:25.765785+0000 mgr.y (mgr.14397) 52 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-09-13T00:30:27.311 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[24895]: audit 2024-09-13T00:30:26.072333+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:27.311 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[24895]: audit 2024-09-13T00:30:26.088313+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:27.311 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[24895]: audit 2024-09-13T00:30:26.101495+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:27.311 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[24895]: audit 2024-09-13T00:30:26.113176+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:27.311 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[24895]: audit 2024-09-13T00:30:26.173173+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:30:27.311 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=settings t=2024-09-13T00:30:27.268542065Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-13T00:30:27.311 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=settings t=2024-09-13T00:30:27.268760849Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-13T00:30:27.311 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=settings t=2024-09-13T00:30:27.268789788Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-13T00:30:27.312 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=settings t=2024-09-13T00:30:27.268805022Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-13T00:30:27.312 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=settings t=2024-09-13T00:30:27.268819306Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T00:30:27.312 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=settings t=2024-09-13T00:30:27.268832675Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T00:30:27.312 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=settings t=2024-09-13T00:30:27.268846343Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T00:30:27.312 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=settings t=2024-09-13T00:30:27.268859784Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T00:30:27.312 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=settings t=2024-09-13T00:30:27.268872936Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-13T00:30:27.312 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=settings t=2024-09-13T00:30:27.268887057Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-13T00:30:27.312 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=settings t=2024-09-13T00:30:27.268900311Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-13T00:30:27.312 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=settings t=2024-09-13T00:30:27.268913267Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-13T00:30:27.312 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=settings t=2024-09-13T00:30:27.268926446Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-13T00:30:27.312 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=settings t=2024-09-13T00:30:27.268939613Z level=info msg="App mode production" 2024-09-13T00:30:27.313 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=sqlstore t=2024-09-13T00:30:27.269014852Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-13T00:30:27.313 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=sqlstore t=2024-09-13T00:30:27.269056036Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T00:30:27.313 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.270055686Z level=info msg="Starting DB migrations" 2024-09-13T00:30:27.313 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.270142849Z level=info msg="Executing migration" id="create migration_log table" 2024-09-13T00:30:27.313 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.314715589Z level=info msg="Executing migration" id="create user table" 2024-09-13T00:30:27.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:27 smithi077 bash[22596]: cluster 2024-09-13T00:30:25.765785+0000 mgr.y (mgr.14397) 52 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-09-13T00:30:27.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:27 smithi077 bash[22596]: audit 2024-09-13T00:30:26.072333+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:27.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:27 smithi077 bash[22596]: audit 2024-09-13T00:30:26.088313+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:27.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:27 smithi077 bash[22596]: audit 2024-09-13T00:30:26.101495+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:27.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:27 smithi077 bash[22596]: audit 2024-09-13T00:30:26.113176+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:27.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:27 smithi077 bash[22596]: audit 2024-09-13T00:30:26.173173+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:30:27.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:27 smithi077 bash[32040]: cluster 2024-09-13T00:30:25.765785+0000 mgr.y (mgr.14397) 52 : cluster [DBG] pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 341 B/s wr, 0 op/s 2024-09-13T00:30:27.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:27 smithi077 bash[32040]: audit 2024-09-13T00:30:26.072333+0000 mon.a (mon.0) 698 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:27.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:27 smithi077 bash[32040]: audit 2024-09-13T00:30:26.088313+0000 mon.a (mon.0) 699 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:27.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:27 smithi077 bash[32040]: audit 2024-09-13T00:30:26.101495+0000 mon.a (mon.0) 700 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:27.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:27 smithi077 bash[32040]: audit 2024-09-13T00:30:26.113176+0000 mon.a (mon.0) 701 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:27.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:27 smithi077 bash[32040]: audit 2024-09-13T00:30:26.173173+0000 mon.c (mon.2) 48 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:30:27.563 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.339845358Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-13T00:30:27.563 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.365262686Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-13T00:30:27.563 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.390498723Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-13T00:30:27.564 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.415686301Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-13T00:30:27.564 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.45781011Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-13T00:30:27.564 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.49995538Z level=info msg="Executing migration" id="create user table v2" 2024-09-13T00:30:27.564 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.541978451Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-13T00:30:27.564 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.567226802Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-13T00:30:27.656 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:30:27.825 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.592868923Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-13T00:30:27.825 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.634814157Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-13T00:30:27.825 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.660107981Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-13T00:30:27.825 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.685359672Z level=info msg="Executing migration" id="Update user table charset" 2024-09-13T00:30:27.825 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.710543747Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-13T00:30:27.825 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.735850371Z level=info msg="Executing migration" id="Add missing user data" 2024-09-13T00:30:27.825 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.777848985Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-13T00:30:27.825 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.803144119Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-13T00:30:27.825 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.828460669Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-13T00:30:28.076 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.870285325Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-13T00:30:28.077 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.912476675Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-13T00:30:28.077 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.937867437Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-13T00:30:28.077 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:27 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:27.979871845Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-13T00:30:28.077 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.005145852Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-13T00:30:28.077 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.030342824Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-13T00:30:28.077 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.080609205Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-13T00:30:28.330 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.105841449Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-13T00:30:28.330 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.131208743Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-13T00:30:28.330 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.156441114Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-13T00:30:28.330 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.181607933Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-13T00:30:28.330 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.198467851Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-13T00:30:28.330 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.215506106Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-13T00:30:28.330 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.249167145Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-13T00:30:28.331 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.266224825Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-13T00:30:28.331 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.299858592Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-13T00:30:28.331 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.316827023Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-13T00:30:28.331 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.333848817Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-13T00:30:28.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.350502519Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-13T00:30:28.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.376151363Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-13T00:30:28.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.409808626Z level=info msg="Executing migration" id="create star table" 2024-09-13T00:30:28.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.451838763Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-13T00:30:28.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.477119434Z level=info msg="Executing migration" id="create org table v1" 2024-09-13T00:30:28.601 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.502299373Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-13T00:30:28.601 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.544291017Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-13T00:30:28.601 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.569622948Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-13T00:30:28.868 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[24895]: cluster 2024-09-13T00:30:27.766358+0000 mgr.y (mgr.14397) 53 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 264 B/s wr, 0 op/s 2024-09-13T00:30:28.869 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[24895]: audit 2024-09-13T00:30:27.833875+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:28.869 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[24895]: audit 2024-09-13T00:30:27.838065+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:28.869 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[24895]: audit 2024-09-13T00:30:27.871350+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:28.869 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.611588772Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-13T00:30:28.869 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.636926593Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-13T00:30:28.869 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.662129294Z level=info msg="Executing migration" id="Update org table charset" 2024-09-13T00:30:28.869 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.679053259Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-13T00:30:28.869 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.695932829Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-13T00:30:28.869 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.721254608Z level=info msg="Executing migration" id="create dashboard table" 2024-09-13T00:30:28.870 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.746468687Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-13T00:30:28.870 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.78859002Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-13T00:30:28.870 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.830579132Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-13T00:30:28.870 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.872695485Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-13T00:30:29.083 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:28 smithi077 bash[32040]: cluster 2024-09-13T00:30:27.766358+0000 mgr.y (mgr.14397) 53 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 264 B/s wr, 0 op/s 2024-09-13T00:30:29.084 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:28 smithi077 bash[32040]: audit 2024-09-13T00:30:27.833875+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:29.084 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:28 smithi077 bash[32040]: audit 2024-09-13T00:30:27.838065+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:29.084 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:28 smithi077 bash[32040]: audit 2024-09-13T00:30:27.871350+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:29.084 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:28 smithi077 bash[22596]: cluster 2024-09-13T00:30:27.766358+0000 mgr.y (mgr.14397) 53 : cluster [DBG] pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 264 B/s wr, 0 op/s 2024-09-13T00:30:29.084 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:28 smithi077 bash[22596]: audit 2024-09-13T00:30:27.833875+0000 mon.a (mon.0) 702 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:29.084 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:28 smithi077 bash[22596]: audit 2024-09-13T00:30:27.838065+0000 mon.c (mon.2) 49 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:29.084 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:28 smithi077 bash[22596]: audit 2024-09-13T00:30:27.871350+0000 mon.a (mon.0) 703 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:29.130 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.91468451Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-13T00:30:29.130 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.93980587Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-13T00:30:29.130 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.965280045Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-13T00:30:29.130 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:28 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:28.990350983Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-13T00:30:29.130 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.007467168Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-13T00:30:29.130 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.032615203Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-13T00:30:29.130 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.057931632Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-13T00:30:29.130 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.083054454Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-13T00:30:29.130 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.099956735Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-13T00:30:29.130 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.116966934Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-13T00:30:29.399 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.134131809Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-13T00:30:29.400 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.159668224Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-13T00:30:29.400 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.184910075Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-13T00:30:29.400 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.21013231Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-13T00:30:29.400 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.252078776Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-13T00:30:29.400 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.294126167Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-13T00:30:29.400 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.335986424Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-13T00:30:29.400 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.377986061Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-13T00:30:29.400 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.403283752Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-13T00:30:29.677 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.42854371Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-13T00:30:29.677 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.470488939Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-13T00:30:29.677 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.49580062Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-13T00:30:29.677 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.52111527Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-13T00:30:29.677 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.546435204Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-13T00:30:29.677 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.588333233Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-13T00:30:29.677 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.61362504Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-13T00:30:29.677 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.638933591Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-13T00:30:29.678 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.680928673Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-13T00:30:29.946 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.706111671Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-13T00:30:29.946 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.731321722Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-13T00:30:29.946 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.773297493Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-13T00:30:29.946 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.798598182Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-13T00:30:29.946 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.840444465Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-13T00:30:29.946 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.899361136Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-13T00:30:29.946 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.924677886Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-13T00:30:29.947 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.949911337Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-13T00:30:30.207 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:29 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:29.97513632Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-13T00:30:30.207 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.000459136Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-13T00:30:30.208 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.042432238Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-13T00:30:30.208 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.068116036Z level=info msg="Executing migration" id="create data_source table" 2024-09-13T00:30:30.208 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.093358754Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-13T00:30:30.208 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.1184959Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-13T00:30:30.208 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.143850483Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-13T00:30:30.208 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.169048745Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-13T00:30:30.208 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.21135225Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-13T00:30:30.468 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.236309621Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-13T00:30:30.469 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.278337239Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-13T00:30:30.469 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.295303511Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-13T00:30:30.469 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.312218104Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-13T00:30:30.469 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.379209333Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-13T00:30:30.469 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.404614796Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-13T00:30:30.469 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.446495386Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-13T00:30:30.469 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.471629457Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-13T00:30:30.737 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.497090937Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-13T00:30:30.737 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.522302239Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-13T00:30:30.737 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.564253942Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-13T00:30:30.737 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.589628002Z level=info msg="Executing migration" id="Add uid column" 2024-09-13T00:30:30.737 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.614855584Z level=info msg="Executing migration" id="Update uid value" 2024-09-13T00:30:30.737 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.640102584Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-13T00:30:30.738 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.665380472Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-13T00:30:30.738 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.690636141Z level=info msg="Executing migration" id="create api_key table" 2024-09-13T00:30:30.738 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.715951698Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-13T00:30:30.738 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.741189312Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-13T00:30:30.998 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:30:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:30:30.999 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[24895]: cluster 2024-09-13T00:30:29.767183+0000 mgr.y (mgr.14397) 54 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 234 B/s wr, 0 op/s 2024-09-13T00:30:30.999 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.783241961Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-13T00:30:31.000 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.808604251Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-13T00:30:31.000 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.850423721Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-13T00:30:31.000 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.892570811Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-13T00:30:31.000 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.909886493Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-13T00:30:31.000 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.935154521Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-13T00:30:31.001 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.952056273Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-13T00:30:31.001 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.96888185Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-13T00:30:31.001 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:30 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:30.985682032Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-13T00:30:31.001 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.002713604Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-13T00:30:31.135 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:30 smithi077 bash[22596]: cluster 2024-09-13T00:30:29.767183+0000 mgr.y (mgr.14397) 54 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 234 B/s wr, 0 op/s 2024-09-13T00:30:31.135 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:30 smithi077 bash[32040]: cluster 2024-09-13T00:30:29.767183+0000 mgr.y (mgr.14397) 54 : cluster [DBG] pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 234 B/s wr, 0 op/s 2024-09-13T00:30:31.284 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.019555646Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-13T00:30:31.284 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.061670068Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-13T00:30:31.285 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.103948795Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-13T00:30:31.285 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.129190912Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-13T00:30:31.285 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.154230587Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-13T00:30:31.285 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.179510973Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-13T00:30:31.285 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.204766281Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-13T00:30:31.285 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.246788279Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-13T00:30:31.285 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.288835176Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-13T00:30:31.489 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:30:31.562 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.330945464Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-13T00:30:31.562 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.373230804Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-13T00:30:31.562 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.398435804Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-13T00:30:31.563 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.440504041Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-13T00:30:31.563 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.48260962Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-13T00:30:31.563 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.524656588Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-13T00:30:31.839 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.56682171Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-13T00:30:31.839 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.59212101Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-13T00:30:31.839 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.617392944Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-13T00:30:31.839 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.642623972Z level=info msg="Executing migration" id="create quota table v1" 2024-09-13T00:30:31.840 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.676708722Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-13T00:30:31.840 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.718464164Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-13T00:30:31.840 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.76049391Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-13T00:30:31.840 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.785936116Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-13T00:30:31.840 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.811147888Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-13T00:30:32.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[24895]: cluster 2024-09-13T00:30:30.843565+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-09-13T00:30:32.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[24895]: audit 2024-09-13T00:30:31.232243+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:32.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[24895]: audit 2024-09-13T00:30:31.238897+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:32.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[24895]: audit 2024-09-13T00:30:31.487677+0000 mon.a (mon.0) 707 : audit [DBG] from='client.? 172.21.15.77:0/1000774140' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T00:30:32.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[24895]: audit 2024-09-13T00:30:31.731541+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:32.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[24895]: audit 2024-09-13T00:30:31.745127+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:32.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[24895]: audit 2024-09-13T00:30:31.747902+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:30:32.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[24895]: audit 2024-09-13T00:30:31.749321+0000 mon.c (mon.2) 51 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:30:32.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[24895]: audit 2024-09-13T00:30:31.758519+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:32.101 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.853111738Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-13T00:30:32.101 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.903544278Z level=info msg="Executing migration" id="create session table" 2024-09-13T00:30:32.101 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.928992501Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-13T00:30:32.101 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.954200392Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-13T00:30:32.101 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:31 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:31.979463831Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-13T00:30:32.101 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.004730688Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-13T00:30:32.101 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.030026676Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-13T00:30:32.102 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.055208605Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-13T00:30:32.102 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.080453441Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-13T00:30:32.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[22596]: cluster 2024-09-13T00:30:30.843565+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-09-13T00:30:32.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[22596]: audit 2024-09-13T00:30:31.232243+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:32.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[22596]: audit 2024-09-13T00:30:31.238897+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:32.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[22596]: audit 2024-09-13T00:30:31.487677+0000 mon.a (mon.0) 707 : audit [DBG] from='client.? 172.21.15.77:0/1000774140' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T00:30:32.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[22596]: audit 2024-09-13T00:30:31.731541+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:32.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[22596]: audit 2024-09-13T00:30:31.745127+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:32.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[22596]: audit 2024-09-13T00:30:31.747902+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:30:32.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[22596]: audit 2024-09-13T00:30:31.749321+0000 mon.c (mon.2) 51 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:30:32.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[22596]: audit 2024-09-13T00:30:31.758519+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:32.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[32040]: cluster 2024-09-13T00:30:30.843565+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e58: 8 total, 8 up, 8 in 2024-09-13T00:30:32.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[32040]: audit 2024-09-13T00:30:31.232243+0000 mon.a (mon.0) 705 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:32.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[32040]: audit 2024-09-13T00:30:31.238897+0000 mon.a (mon.0) 706 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:32.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[32040]: audit 2024-09-13T00:30:31.487677+0000 mon.a (mon.0) 707 : audit [DBG] from='client.? 172.21.15.77:0/1000774140' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T00:30:32.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[32040]: audit 2024-09-13T00:30:31.731541+0000 mon.a (mon.0) 708 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:32.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[32040]: audit 2024-09-13T00:30:31.745127+0000 mon.a (mon.0) 709 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:32.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[32040]: audit 2024-09-13T00:30:31.747902+0000 mon.c (mon.2) 50 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:30:32.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[32040]: audit 2024-09-13T00:30:31.749321+0000 mon.c (mon.2) 51 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:30:32.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:31 smithi077 bash[32040]: audit 2024-09-13T00:30:31.758519+0000 mon.a (mon.0) 710 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:32.396 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.129612967Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-13T00:30:32.396 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.156234318Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-13T00:30:32.396 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.1814674Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-13T00:30:32.396 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.206687551Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-13T00:30:32.396 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.248684234Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-13T00:30:32.396 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.290622381Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-13T00:30:32.396 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.332665363Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-13T00:30:32.396 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.374663229Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-13T00:30:32.408 INFO:teuthology.orchestra.run.smithi077.stdout:{"epoch":20,"flags":0,"active_gid":14397,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6800","nonce":930711876},{"type":"v1","addr":"172.21.15.77:6801","nonce":930711876}]},"active_addr":"172.21.15.77:6801/930711876","active_change":"2024-09-13T00:29:27.728667+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24289,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.77:8443/","prometheus":"http://172.21.15.77:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.77:0","nonce":1115799687}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.77:0","nonce":3515285019}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.77:0","nonce":2219740569}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.77:0","nonce":676410089}]}]} 2024-09-13T00:30:32.414 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T00:30:32.414 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T00:30:32.414 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T00:30:32.657 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.399967426Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-13T00:30:32.657 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.425284131Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-13T00:30:32.658 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.450494085Z level=info msg="Executing migration" id="create alert table v1" 2024-09-13T00:30:32.658 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.492468535Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-13T00:30:32.658 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.517707881Z level=info msg="Executing migration" id="add index alert state" 2024-09-13T00:30:32.658 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.559885168Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-13T00:30:32.658 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.576877359Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-13T00:30:32.658 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.593805372Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-13T00:30:32.658 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.610545919Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-13T00:30:32.658 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.627555491Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-13T00:30:32.658 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.644447887Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-13T00:30:32.658 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.661372446Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-13T00:30:32.909 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[24895]: cluster 2024-09-13T00:30:31.767809+0000 mgr.y (mgr.14397) 55 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 210 B/s wr, 0 op/s 2024-09-13T00:30:32.910 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[24895]: cephadm 2024-09-13T00:30:31.813578+0000 mgr.y (mgr.14397) 56 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T00:30:32.910 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[24895]: cephadm 2024-09-13T00:30:31.822606+0000 mgr.y (mgr.14397) 57 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi077 2024-09-13T00:30:32.910 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.686539689Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-13T00:30:32.910 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.711781987Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-13T00:30:32.910 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.737111815Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-13T00:30:32.910 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.779135114Z level=info msg="Executing migration" id="Add column is_default" 2024-09-13T00:30:32.910 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.804364756Z level=info msg="Executing migration" id="Add column frequency" 2024-09-13T00:30:32.910 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.846278932Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-13T00:30:32.910 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.888400876Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-13T00:30:32.910 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.913698851Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-13T00:30:33.179 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.955669377Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-13T00:30:33.179 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:32 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:32.980881419Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-13T00:30:33.179 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.006196257Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-13T00:30:33.179 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.031520101Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-13T00:30:33.179 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.073548314Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-13T00:30:33.179 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.098807669Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-13T00:30:33.180 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.132314954Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-13T00:30:33.180 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.157621405Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-13T00:30:33.180 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.18283457Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-13T00:30:33.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:32 smithi077 bash[22596]: cluster 2024-09-13T00:30:31.767809+0000 mgr.y (mgr.14397) 55 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 210 B/s wr, 0 op/s 2024-09-13T00:30:33.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:32 smithi077 bash[22596]: cephadm 2024-09-13T00:30:31.813578+0000 mgr.y (mgr.14397) 56 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T00:30:33.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:32 smithi077 bash[22596]: cephadm 2024-09-13T00:30:31.822606+0000 mgr.y (mgr.14397) 57 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi077 2024-09-13T00:30:33.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:32 smithi077 bash[32040]: cluster 2024-09-13T00:30:31.767809+0000 mgr.y (mgr.14397) 55 : cluster [DBG] pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 210 B/s wr, 0 op/s 2024-09-13T00:30:33.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:32 smithi077 bash[32040]: cephadm 2024-09-13T00:30:31.813578+0000 mgr.y (mgr.14397) 56 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T00:30:33.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:32 smithi077 bash[32040]: cephadm 2024-09-13T00:30:31.822606+0000 mgr.y (mgr.14397) 57 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi077 2024-09-13T00:30:33.431 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.22475548Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-13T00:30:33.432 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.241637997Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-13T00:30:33.432 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.25853309Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-13T00:30:33.432 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.275498278Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-13T00:30:33.432 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.292385145Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-13T00:30:33.432 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.309281957Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-13T00:30:33.432 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.326222089Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-13T00:30:33.432 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.36844954Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-13T00:30:33.432 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.39364118Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-13T00:30:33.692 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.435636346Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-13T00:30:33.692 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.460842884Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-13T00:30:33.692 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.486097239Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-13T00:30:33.692 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.528062812Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-13T00:30:33.692 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.553373654Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-13T00:30:33.692 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.59528266Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-13T00:30:33.693 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.620489903Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-13T00:30:33.693 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.64579099Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-13T00:30:33.693 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.670984516Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-13T00:30:33.693 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.696240271Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-13T00:30:33.920 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:33 smithi077 bash[22596]: audit 2024-09-13T00:30:32.921760+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:33.969 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[24895]: audit 2024-09-13T00:30:32.921760+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:33.969 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.72149777Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-13T00:30:33.970 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.746636322Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-13T00:30:33.970 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.788683508Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-13T00:30:33.970 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.839090896Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-13T00:30:33.970 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.864534353Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-13T00:30:33.970 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.906370553Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-13T00:30:33.970 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.973569649Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-13T00:30:34.255 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:33 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:33.998808227Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-13T00:30:34.255 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.023985193Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-13T00:30:34.256 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.049264938Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-13T00:30:34.256 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.074398432Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-13T00:30:34.256 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.099749339Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-13T00:30:34.256 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.125028207Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-13T00:30:34.256 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.150220669Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-13T00:30:34.256 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.175518498Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-13T00:30:34.256 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.217527591Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-13T00:30:34.256 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.259460728Z level=info msg="Executing migration" id="Move region to single row" 2024-09-13T00:30:34.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:33 smithi077 bash[32040]: audit 2024-09-13T00:30:32.921760+0000 mon.a (mon.0) 711 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:34.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.301703312Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-13T00:30:34.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.343681835Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-13T00:30:34.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.385734047Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-13T00:30:34.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.411126525Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-13T00:30:34.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.436320704Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-13T00:30:34.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.478271804Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-13T00:30:34.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.503656718Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-13T00:30:34.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.545620424Z level=info msg="Executing migration" id="create test_data table" 2024-09-13T00:30:34.802 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.571016281Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-13T00:30:34.802 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.612901036Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-13T00:30:34.802 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.63824033Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-13T00:30:34.803 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.663331323Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-13T00:30:34.803 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.688621781Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-13T00:30:34.803 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.713846001Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-13T00:30:34.803 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.739156805Z level=info msg="Executing migration" id="create team table" 2024-09-13T00:30:34.803 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.781103909Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-13T00:30:34.803 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.806280738Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-13T00:30:35.054 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[24895]: cluster 2024-09-13T00:30:33.768226+0000 mgr.y (mgr.14397) 58 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T00:30:35.054 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.831514607Z level=info msg="Executing migration" id="create team member table" 2024-09-13T00:30:35.054 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.856813695Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-13T00:30:35.055 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.898732972Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-13T00:30:35.055 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.940806818Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-13T00:30:35.055 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.966064779Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-13T00:30:35.055 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:34 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:34.991267298Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-13T00:30:35.055 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.033233329Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-13T00:30:35.055 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.058453201Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-13T00:30:35.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:34 smithi077 bash[22596]: cluster 2024-09-13T00:30:33.768226+0000 mgr.y (mgr.14397) 58 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T00:30:35.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:34 smithi077 bash[32040]: cluster 2024-09-13T00:30:33.768226+0000 mgr.y (mgr.14397) 58 : cluster [DBG] pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-13T00:30:35.307 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.100510244Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-13T00:30:35.307 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.125878595Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-13T00:30:35.307 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.167763439Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-13T00:30:35.307 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.19313388Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-13T00:30:35.307 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.235287582Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-13T00:30:35.307 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.260520981Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-13T00:30:35.307 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.285669234Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-13T00:30:35.307 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.310969836Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-13T00:30:35.593 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.336225404Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-13T00:30:35.593 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.378235058Z level=info msg="Executing migration" id="create tag table" 2024-09-13T00:30:35.593 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.403538972Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-13T00:30:35.593 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.428795208Z level=info msg="Executing migration" id="create login attempt table" 2024-09-13T00:30:35.593 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.454094639Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-13T00:30:35.593 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.479286239Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-13T00:30:35.593 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.521207456Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-13T00:30:35.593 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.554835633Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-13T00:30:35.593 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.596812127Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-13T00:30:35.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.622102068Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-13T00:30:35.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.647339234Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-13T00:30:35.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.672621509Z level=info msg="Executing migration" id="create user auth table" 2024-09-13T00:30:35.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.697844054Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-13T00:30:35.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.723136233Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-13T00:30:35.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.748390625Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-13T00:30:35.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.790283776Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-13T00:30:35.851 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.815633051Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-13T00:30:35.851 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.840870093Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-13T00:30:36.140 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.882908726Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-13T00:30:36.140 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.924876959Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-13T00:30:36.140 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.966887606Z level=info msg="Executing migration" id="create server_lock table" 2024-09-13T00:30:36.140 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:35 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:35.992186464Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-13T00:30:36.140 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.017447916Z level=info msg="Executing migration" id="create user auth token table" 2024-09-13T00:30:36.140 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.042647925Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-13T00:30:36.140 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.067906575Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-13T00:30:36.140 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.09317651Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-13T00:30:36.141 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.143682061Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-13T00:30:36.401 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.16064473Z level=info msg="Executing migration" id="create cache_data table" 2024-09-13T00:30:36.401 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.194298167Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-13T00:30:36.401 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.211281577Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-13T00:30:36.401 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.228202989Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-13T00:30:36.401 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.245101596Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-13T00:30:36.401 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.270219939Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-13T00:30:36.401 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.312310139Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-13T00:30:36.402 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.362706578Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-13T00:30:36.402 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.404595809Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-13T00:30:36.678 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.430056822Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-13T00:30:36.678 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.455222479Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-13T00:30:36.678 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.480535354Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-13T00:30:36.679 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.50568489Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-13T00:30:36.679 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.547732206Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-13T00:30:36.679 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.573028981Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-13T00:30:36.679 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.598252841Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-13T00:30:36.679 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.640289758Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-13T00:30:36.679 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.682179208Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-13T00:30:36.925 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:36 smithi077 systemd[1]: Stopping Ceph alertmanager.a for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:30:36.925 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:36 smithi077 bash[56627]: ts=2024-09-13T00:30:36.795Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T00:30:36.946 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[24895]: cluster 2024-09-13T00:30:35.768828+0000 mgr.y (mgr.14397) 59 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:36.947 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.707415401Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-13T00:30:36.947 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.732686733Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-13T00:30:36.947 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.774621547Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-13T00:30:36.947 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.799881859Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-13T00:30:36.947 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.841909823Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-13T00:30:36.947 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.867148378Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-13T00:30:36.947 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.909247191Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-13T00:30:36.947 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:36 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:36.95104049Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-13T00:30:37.183 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:30:37.265 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.024147589Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-13T00:30:37.265 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.068851276Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-13T00:30:37.265 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.144099071Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-13T00:30:37.274 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:36 smithi077 bash[32040]: cluster 2024-09-13T00:30:35.768828+0000 mgr.y (mgr.14397) 59 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:37.274 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:36 smithi077 bash[22596]: cluster 2024-09-13T00:30:35.768828+0000 mgr.y (mgr.14397) 59 : cluster [DBG] pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:37.516 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.269438152Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-13T00:30:37.516 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.311554625Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-13T00:30:37.516 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.328442922Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-13T00:30:37.516 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.345376761Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-13T00:30:37.516 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.362342844Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-13T00:30:37.516 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.379234Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-13T00:30:37.516 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.412896022Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-13T00:30:37.516 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.429852507Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-13T00:30:37.516 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.455055487Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-13T00:30:37.526 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:37 smithi077 bash[58166]: ceph-9218306c-7165-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T00:30:37.770 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.519850947Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-13T00:30:37.770 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.547392707Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-13T00:30:37.770 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.589359474Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-13T00:30:37.770 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.631365771Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-13T00:30:37.771 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.656666094Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-13T00:30:37.771 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.681938719Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-13T00:30:37.771 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.707215485Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-13T00:30:37.771 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.732442813Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-13T00:30:37.771 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.774416914Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-13T00:30:38.048 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.816459177Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-13T00:30:38.048 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.84175144Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-13T00:30:38.048 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.867068141Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-13T00:30:38.048 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.909026785Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-13T00:30:38.049 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.950979193Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-13T00:30:38.049 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:37 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:37.976280941Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-13T00:30:38.049 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:38 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:38.018100804Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-13T00:30:38.276 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:37 smithi077 bash[58236]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T00:30:38.276 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:37 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T00:30:38.276 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:37 smithi077 systemd[1]: Stopped Ceph alertmanager.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:30:38.277 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:37 smithi077 systemd[1]: Started Ceph alertmanager.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:30:38.334 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:38 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:38.052176961Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-13T00:30:38.334 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:38 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:38.094230554Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-13T00:30:38.334 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:38 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:38.119481937Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-13T00:30:38.335 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:38 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:38.144714473Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-13T00:30:38.335 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:38 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:38.186643961Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-13T00:30:38.335 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:38 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:38.228492518Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-13T00:30:38.335 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:38 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:38.279678622Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-13T00:30:38.335 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:38 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:38.338284214Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-13T00:30:38.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:38 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:38.530081881Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-13T00:30:39.100 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:38 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:38.819188982Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-13T00:30:39.100 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:38 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:38.950018061Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-13T00:30:39.100 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:39.026191815Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-13T00:30:39.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[24895]: cluster 2024-09-13T00:30:37.769486+0000 mgr.y (mgr.14397) 60 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:39.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[24895]: audit 2024-09-13T00:30:38.004562+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:39.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[24895]: audit 2024-09-13T00:30:38.010854+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:39.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[24895]: cephadm 2024-09-13T00:30:38.012397+0000 mgr.y (mgr.14397) 61 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T00:30:39.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[24895]: cephadm 2024-09-13T00:30:38.020751+0000 mgr.y (mgr.14397) 62 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi088 2024-09-13T00:30:39.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:39 smithi077 bash[22596]: cluster 2024-09-13T00:30:37.769486+0000 mgr.y (mgr.14397) 60 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:39.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:39 smithi077 bash[22596]: audit 2024-09-13T00:30:38.004562+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:39.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:39 smithi077 bash[22596]: audit 2024-09-13T00:30:38.010854+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:39.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:39 smithi077 bash[22596]: cephadm 2024-09-13T00:30:38.012397+0000 mgr.y (mgr.14397) 61 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T00:30:39.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:39 smithi077 bash[22596]: cephadm 2024-09-13T00:30:38.020751+0000 mgr.y (mgr.14397) 62 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi088 2024-09-13T00:30:39.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:39 smithi077 bash[32040]: cluster 2024-09-13T00:30:37.769486+0000 mgr.y (mgr.14397) 60 : cluster [DBG] pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:39.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:39 smithi077 bash[32040]: audit 2024-09-13T00:30:38.004562+0000 mon.a (mon.0) 712 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:39.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:39 smithi077 bash[32040]: audit 2024-09-13T00:30:38.010854+0000 mon.a (mon.0) 713 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:39.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:39 smithi077 bash[32040]: cephadm 2024-09-13T00:30:38.012397+0000 mgr.y (mgr.14397) 61 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T00:30:39.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:39 smithi077 bash[32040]: cephadm 2024-09-13T00:30:38.020751+0000 mgr.y (mgr.14397) 62 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi088 2024-09-13T00:30:39.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:39.31053843Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-13T00:30:39.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:39.436193023Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-13T00:30:39.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:39.52860267Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-13T00:30:39.890 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:39.629101507Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-13T00:30:39.890 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:39.713030422Z level=info msg="Executing migration" id="create alert_image table" 2024-09-13T00:30:39.890 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:39.763419246Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-13T00:30:39.890 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:39.827854417Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-13T00:30:39.891 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:39.89439874Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-13T00:30:40.163 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:39.931398273Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-13T00:30:40.163 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:39.973583264Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-13T00:30:40.163 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:39 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:39.973831292Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-13T00:30:40.163 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:40 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:40.024194296Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-13T00:30:40.163 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:40 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:40.09159736Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-13T00:30:40.527 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:40 smithi077 bash[58266]: ts=2024-09-13T00:30:40.147Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-13T00:30:40.527 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:40 smithi077 bash[58266]: ts=2024-09-13T00:30:40.147Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-13T00:30:40.527 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:40 smithi077 bash[58266]: ts=2024-09-13T00:30:40.149Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.77 port=9094 2024-09-13T00:30:40.527 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:40 smithi077 bash[58266]: ts=2024-09-13T00:30:40.149Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T00:30:40.527 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:40 smithi077 bash[58266]: ts=2024-09-13T00:30:40.180Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T00:30:40.527 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:40 smithi077 bash[58266]: ts=2024-09-13T00:30:40.180Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T00:30:40.527 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:40 smithi077 bash[58266]: ts=2024-09-13T00:30:40.183Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-13T00:30:40.527 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:40 smithi077 bash[58266]: ts=2024-09-13T00:30:40.183Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T00:30:40.538 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:40 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:40.167154045Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-13T00:30:40.538 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:40 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:40.167404939Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-13T00:30:40.538 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:40 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:40.167649683Z level=info msg="alerts found to migrate" alerts=0 2024-09-13T00:30:40.538 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:40 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:40.252295308Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-13T00:30:40.538 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:40 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:40.360710121Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-13T00:30:40.799 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:30:40 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:30:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:30:40.799 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:40 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:40.539186381Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-13T00:30:41.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:41 smithi088 bash[24895]: cluster 2024-09-13T00:30:39.770180+0000 mgr.y (mgr.14397) 63 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:41.100 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:40 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:40.803335724Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-13T00:30:41.100 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:40 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:40.870859388Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-13T00:30:41.100 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:40 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:40.938282194Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-13T00:30:41.100 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:40 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:40.997644501Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-13T00:30:41.101 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:41 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:41.039816284Z level=info msg="Executing migration" id="create data_keys table" 2024-09-13T00:30:41.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:41 smithi077 bash[22596]: cluster 2024-09-13T00:30:39.770180+0000 mgr.y (mgr.14397) 63 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:41.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:41 smithi077 bash[32040]: cluster 2024-09-13T00:30:39.770180+0000 mgr.y (mgr.14397) 63 : cluster [DBG] pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:41.516 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:41 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:41.132317578Z level=info msg="Executing migration" id="create secrets table" 2024-09-13T00:30:41.516 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:41 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:41.325991163Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-13T00:30:41.516 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:41 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:41.520248458Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-13T00:30:41.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:41 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:41.643242494Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-13T00:30:41.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:41 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:41.730778414Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-13T00:30:42.201 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:41 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:41.933323899Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-13T00:30:42.201 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:42 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:42.054387845Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-13T00:30:42.202 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:42 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:42.118542573Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-13T00:30:42.202 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:42 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:42.205420258Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-13T00:30:42.276 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:42 smithi077 bash[58266]: ts=2024-09-13T00:30:42.150Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00065811s 2024-09-13T00:30:42.345 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:30:42.345 INFO:teuthology.orchestra.run.smithi077.stdout:{"epoch":58,"fsid":"9218306c-7165-11ef-bceb-c7b262605968","created":"2024-09-13T00:18:59.375829+0000","modified":"2024-09-13T00:30:30.837095+0000","last_up_change":"2024-09-13T00:28:29.926728+0000","last_in_change":"2024-09-13T00:28:05.097746+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T00:24:15.726444+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T00:30:09.880408+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":57,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"e0860feb-72c9-4c49-933d-0a49b68caeee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6802","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6803","nonce":1623816886}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6804","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6805","nonce":1623816886}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6808","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6809","nonce":1623816886}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6806","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6807","nonce":1623816886}]},"public_addr":"172.21.15.77:6803/1623816886","cluster_addr":"172.21.15.77:6805/1623816886","heartbeat_back_addr":"172.21.15.77:6809/1623816886","heartbeat_front_addr":"172.21.15.77:6807/1623816886","state":["exists","up"]},{"osd":1,"uuid":"7ee01388-61b0-4a60-81f2-08c3c072ec0e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6810","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6811","nonce":1691613782}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6812","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6813","nonce":1691613782}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6816","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6817","nonce":1691613782}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6814","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6815","nonce":1691613782}]},"public_addr":"172.21.15.77:6811/1691613782","cluster_addr":"172.21.15.77:6813/1691613782","heartbeat_back_addr":"172.21.15.77:6817/1691613782","heartbeat_front_addr":"172.21.15.77:6815/1691613782","state":["exists","up"]},{"osd":2,"uuid":"ff8bc88a-5b11-469f-912d-d3149036b450","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6818","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6819","nonce":2379210664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6820","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6821","nonce":2379210664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6824","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6825","nonce":2379210664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6822","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6823","nonce":2379210664}]},"public_addr":"172.21.15.77:6819/2379210664","cluster_addr":"172.21.15.77:6821/2379210664","heartbeat_back_addr":"172.21.15.77:6825/2379210664","heartbeat_front_addr":"172.21.15.77:6823/2379210664","state":["exists","up"]},{"osd":3,"uuid":"cb032137-609e-41f6-bfc7-e70b77ea9456","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6826","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6827","nonce":2636296765}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6828","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6829","nonce":2636296765}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6832","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6833","nonce":2636296765}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6830","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6831","nonce":2636296765}]},"public_addr":"172.21.15.77:6827/2636296765","cluster_addr":"172.21.15.77:6829/2636296765","heartbeat_back_addr":"172.21.15.77:6833/2636296765","heartbeat_front_addr":"172.21.15.77:6831/2636296765","state":["exists","up"]},{"osd":4,"uuid":"cd38603e-5cdc-42b6-b8c8-656d44df0ed5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6800","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6801","nonce":3645723145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6802","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6803","nonce":3645723145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6806","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6807","nonce":3645723145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6804","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6805","nonce":3645723145}]},"public_addr":"172.21.15.88:6801/3645723145","cluster_addr":"172.21.15.88:6803/3645723145","heartbeat_back_addr":"172.21.15.88:6807/3645723145","heartbeat_front_addr":"172.21.15.88:6805/3645723145","state":["exists","up"]},{"osd":5,"uuid":"d4b81ff3-f257-4f63-a7a7-cbd1ff754544","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6808","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6809","nonce":2699666173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6810","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6811","nonce":2699666173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6814","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6815","nonce":2699666173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6812","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6813","nonce":2699666173}]},"public_addr":"172.21.15.88:6809/2699666173","cluster_addr":"172.21.15.88:6811/2699666173","heartbeat_back_addr":"172.21.15.88:6815/2699666173","heartbeat_front_addr":"172.21.15.88:6813/2699666173","state":["exists","up"]},{"osd":6,"uuid":"31c0d169-c329-440d-848b-057fbbf418ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6816","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6817","nonce":412460705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6818","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6819","nonce":412460705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6822","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6823","nonce":412460705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6820","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6821","nonce":412460705}]},"public_addr":"172.21.15.88:6817/412460705","cluster_addr":"172.21.15.88:6819/412460705","heartbeat_back_addr":"172.21.15.88:6823/412460705","heartbeat_front_addr":"172.21.15.88:6821/412460705","state":["exists","up"]},{"osd":7,"uuid":"2b2c1a7c-3203-4e9a-b07b-4eaae7e09c0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6824","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6825","nonce":2009347492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6826","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6827","nonce":2009347492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6830","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6831","nonce":2009347492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6828","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6829","nonce":2009347492}]},"public_addr":"172.21.15.88:6825/2009347492","cluster_addr":"172.21.15.88:6827/2009347492","heartbeat_back_addr":"172.21.15.88:6831/2009347492","heartbeat_front_addr":"172.21.15.88:6829/2009347492","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:22:21.643909+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:23:17.132405+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:24:13.726010+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:25:11.211228+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:26:00.949032+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:26:49.596019+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:27:38.694576+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:28:28.383054+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.77:0/1008984793":"2024-09-14T00:29:27.728349+0000","172.21.15.77:6801/395417143":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/892433806":"2024-09-14T00:19:25.064947+0000","172.21.15.77:0/4291417054":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6800/426595304":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6801/426595304":"2024-09-14T00:19:25.064947+0000","172.21.15.77:0/1137737372":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6800/395417143":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/1897411349":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/4117010497":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/2198980039":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/1347196058":"2024-09-14T00:29:27.728349+0000","172.21.15.77:6800/3213372405":"2024-09-14T00:19:55.144762+0000","172.21.15.77:6801/3213372405":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/1177255174":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/3827884320":"2024-09-14T00:29:27.728349+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T00:30:42.478 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:42 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:42.294569588Z level=info msg="Executing migration" id="create permission table" 2024-09-13T00:30:42.478 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:42 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:42.347860834Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-13T00:30:42.478 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:42 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:42.418948989Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-13T00:30:42.478 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:42 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:42.482165069Z level=info msg="Executing migration" id="create role table" 2024-09-13T00:30:42.739 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:42 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:42.524201245Z level=info msg="Executing migration" id="add column display_name" 2024-09-13T00:30:42.739 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:42 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:42.566417002Z level=info msg="Executing migration" id="add column group_name" 2024-09-13T00:30:42.739 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:42 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:42.616889908Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-13T00:30:42.739 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:42 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:42.684512438Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-13T00:30:43.009 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:42 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:42.743719924Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-13T00:30:43.010 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:42 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:42.836122419Z level=info msg="Executing migration" id="create team role table" 2024-09-13T00:30:43.010 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:42 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:42.942486883Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-13T00:30:43.120 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T00:30:43.121 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T00:30:43.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:43 smithi077 bash[32040]: cluster 2024-09-13T00:30:41.770783+0000 mgr.y (mgr.14397) 64 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:43.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:43 smithi077 bash[32040]: audit 2024-09-13T00:30:42.348316+0000 mon.a (mon.0) 714 : audit [DBG] from='client.? 172.21.15.77:0/3543218702' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:30:43.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:43 smithi077 bash[32040]: audit 2024-09-13T00:30:42.827197+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:43.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:43 smithi077 bash[22596]: cluster 2024-09-13T00:30:41.770783+0000 mgr.y (mgr.14397) 64 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:43.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:43 smithi077 bash[22596]: audit 2024-09-13T00:30:42.348316+0000 mon.a (mon.0) 714 : audit [DBG] from='client.? 172.21.15.77:0/3543218702' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:30:43.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:43 smithi077 bash[22596]: audit 2024-09-13T00:30:42.827197+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:43.333 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:43.11992471Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-13T00:30:43.334 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[24895]: cluster 2024-09-13T00:30:41.770783+0000 mgr.y (mgr.14397) 64 : cluster [DBG] pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:43.334 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[24895]: audit 2024-09-13T00:30:42.348316+0000 mon.a (mon.0) 714 : audit [DBG] from='client.? 172.21.15.77:0/3543218702' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:30:43.334 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[24895]: audit 2024-09-13T00:30:42.827197+0000 mon.c (mon.2) 52 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:43.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:43.337649248Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-13T00:30:43.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:43.455161176Z level=info msg="Executing migration" id="create user role table" 2024-09-13T00:30:43.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:43.522424437Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-13T00:30:43.929 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:43 smithi088 systemd[1]: Stopping Ceph prometheus.a for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:30:43.929 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[49366]: ts=2024-09-13T00:30:43.799Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T00:30:43.929 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[49366]: ts=2024-09-13T00:30:43.799Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T00:30:43.929 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[49366]: ts=2024-09-13T00:30:43.799Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T00:30:43.929 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[49366]: ts=2024-09-13T00:30:43.799Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T00:30:43.929 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[49366]: ts=2024-09-13T00:30:43.799Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T00:30:43.929 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[49366]: ts=2024-09-13T00:30:43.799Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T00:30:43.929 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[49366]: ts=2024-09-13T00:30:43.799Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T00:30:43.930 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[49366]: ts=2024-09-13T00:30:43.799Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T00:30:43.930 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[49366]: ts=2024-09-13T00:30:43.800Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T00:30:43.930 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[49366]: ts=2024-09-13T00:30:43.800Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T00:30:43.930 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[49366]: ts=2024-09-13T00:30:43.800Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T00:30:43.930 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[49366]: ts=2024-09-13T00:30:43.800Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T00:30:43.930 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:43.615274402Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-13T00:30:43.930 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:43.65722347Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-13T00:30:43.930 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:43.710328891Z level=info msg="Executing migration" id="create builtin role table" 2024-09-13T00:30:43.931 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:43.775289977Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-13T00:30:43.931 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:43.817495378Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-13T00:30:43.931 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:43.859409925Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-13T00:30:43.931 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:43 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:43.932716053Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-13T00:30:44.288 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:44 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:44.018981345Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-13T00:30:44.288 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:44 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:44.127722721Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-13T00:30:44.599 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:44 smithi088 bash[51661]: ceph-9218306c-7165-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T00:30:44.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:44 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:44.289745534Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-13T00:30:44.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:44 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:44.531232631Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-13T00:30:44.862 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:44 smithi088 bash[51722]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T00:30:44.862 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:44 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T00:30:44.862 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:44 smithi088 systemd[1]: Stopped Ceph prometheus.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:30:44.862 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:44 smithi088 systemd[1]: Started Ceph prometheus.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:30:44.862 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:44 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:44.612862359Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-13T00:30:44.862 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:44 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:44.70545627Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-13T00:30:44.862 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:44 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:44.773251274Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-13T00:30:44.862 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:44 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:44.798626312Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-13T00:30:45.162 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:44 smithi077 bash[22845]: [13/Sep/2024:00:30:44] ENGINE Bus STOPPING 2024-09-13T00:30:45.163 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22596]: cluster 2024-09-13T00:30:43.771374+0000 mgr.y (mgr.14397) 65 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:45.163 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22596]: audit 2024-09-13T00:30:44.802407+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:45.163 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22596]: audit 2024-09-13T00:30:44.816788+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:45.163 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22596]: audit 2024-09-13T00:30:44.829550+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T00:30:45.163 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22596]: audit 2024-09-13T00:30:44.832346+0000 mon.c (mon.2) 54 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi077.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T00:30:45.164 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22596]: audit 2024-09-13T00:30:44.843175+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:45.164 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22596]: audit 2024-09-13T00:30:44.860000+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:30:45.164 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22596]: audit 2024-09-13T00:30:44.868183+0000 mon.c (mon.2) 56 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi088.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T00:30:45.164 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22596]: audit 2024-09-13T00:30:44.882699+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:45.164 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22596]: audit 2024-09-13T00:30:44.901781+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T00:30:45.164 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22596]: audit 2024-09-13T00:30:44.909446+0000 mon.c (mon.2) 58 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi088.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T00:30:45.164 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22596]: audit 2024-09-13T00:30:44.920176+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:45.164 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22596]: audit 2024-09-13T00:30:44.931190+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:30:45.164 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[32040]: cluster 2024-09-13T00:30:43.771374+0000 mgr.y (mgr.14397) 65 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:45.164 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[32040]: audit 2024-09-13T00:30:44.802407+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:45.164 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[32040]: audit 2024-09-13T00:30:44.816788+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:45.165 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[32040]: audit 2024-09-13T00:30:44.829550+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T00:30:45.165 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[32040]: audit 2024-09-13T00:30:44.832346+0000 mon.c (mon.2) 54 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi077.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T00:30:45.165 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[32040]: audit 2024-09-13T00:30:44.843175+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:45.165 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[32040]: audit 2024-09-13T00:30:44.860000+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:30:45.165 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[32040]: audit 2024-09-13T00:30:44.868183+0000 mon.c (mon.2) 56 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi088.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T00:30:45.165 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[32040]: audit 2024-09-13T00:30:44.882699+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:45.165 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[32040]: audit 2024-09-13T00:30:44.901781+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T00:30:45.165 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[32040]: audit 2024-09-13T00:30:44.909446+0000 mon.c (mon.2) 58 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi088.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T00:30:45.165 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[32040]: audit 2024-09-13T00:30:44.920176+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:45.165 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[32040]: audit 2024-09-13T00:30:44.931190+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:30:45.269 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:44 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:44.866182646Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-13T00:30:45.269 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:44 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:44.989150588Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-13T00:30:45.269 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[24895]: cluster 2024-09-13T00:30:43.771374+0000 mgr.y (mgr.14397) 65 : cluster [DBG] pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:45.270 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[24895]: audit 2024-09-13T00:30:44.802407+0000 mon.a (mon.0) 715 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:45.270 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[24895]: audit 2024-09-13T00:30:44.816788+0000 mon.a (mon.0) 716 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:45.270 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[24895]: audit 2024-09-13T00:30:44.829550+0000 mon.c (mon.2) 53 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T00:30:45.270 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[24895]: audit 2024-09-13T00:30:44.832346+0000 mon.c (mon.2) 54 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi077.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T00:30:45.270 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[24895]: audit 2024-09-13T00:30:44.843175+0000 mon.a (mon.0) 717 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:45.270 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[24895]: audit 2024-09-13T00:30:44.860000+0000 mon.c (mon.2) 55 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:30:45.270 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[24895]: audit 2024-09-13T00:30:44.868183+0000 mon.c (mon.2) 56 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi088.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T00:30:45.270 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[24895]: audit 2024-09-13T00:30:44.882699+0000 mon.a (mon.0) 718 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:45.270 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[24895]: audit 2024-09-13T00:30:44.901781+0000 mon.c (mon.2) 57 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T00:30:45.270 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[24895]: audit 2024-09-13T00:30:44.909446+0000 mon.c (mon.2) 58 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi088.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T00:30:45.270 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[24895]: audit 2024-09-13T00:30:44.920176+0000 mon.a (mon.0) 719 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:45.270 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[24895]: audit 2024-09-13T00:30:44.931190+0000 mon.c (mon.2) 59 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:30:45.526 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22845]: [13/Sep/2024:00:30:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:30:45.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22845]: [13/Sep/2024:00:30:45] ENGINE Bus STOPPED 2024-09-13T00:30:45.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22845]: [13/Sep/2024:00:30:45] ENGINE Bus STARTING 2024-09-13T00:30:45.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22845]: [13/Sep/2024:00:30:45] ENGINE Serving on http://:::9283 2024-09-13T00:30:45.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22845]: [13/Sep/2024:00:30:45] ENGINE Bus STARTED 2024-09-13T00:30:45.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22845]: [13/Sep/2024:00:30:45] ENGINE Bus STOPPING 2024-09-13T00:30:45.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:45.273864033Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-13T00:30:45.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:45.429628999Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-13T00:30:45.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:45.488364254Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-13T00:30:45.988 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:45.69319699Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-13T00:30:45.989 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:45.823685034Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-13T00:30:45.989 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:45.901085183Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-13T00:30:45.989 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:45 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:45.992788337Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-13T00:30:46.026 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22845]: [13/Sep/2024:00:30:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:30:46.026 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22845]: [13/Sep/2024:00:30:45] ENGINE Bus STOPPED 2024-09-13T00:30:46.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22845]: [13/Sep/2024:00:30:45] ENGINE Bus STARTING 2024-09-13T00:30:46.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22845]: [13/Sep/2024:00:30:45] ENGINE Serving on http://:::9283 2024-09-13T00:30:46.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22845]: [13/Sep/2024:00:30:45] ENGINE Bus STARTED 2024-09-13T00:30:46.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:45 smithi077 bash[22845]: [13/Sep/2024:00:30:45] ENGINE Bus STOPPING 2024-09-13T00:30:46.272 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.092698862Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-13T00:30:46.273 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.142126866Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-13T00:30:46.273 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.209819892Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-13T00:30:46.273 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[24895]: audit 2024-09-13T00:30:44.830290+0000 mgr.y (mgr.14397) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T00:30:46.273 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[24895]: audit 2024-09-13T00:30:44.833208+0000 mgr.y (mgr.14397) 67 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi077.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T00:30:46.273 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[24895]: audit 2024-09-13T00:30:44.861070+0000 mgr.y (mgr.14397) 68 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:30:46.273 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[24895]: audit 2024-09-13T00:30:44.869205+0000 mgr.y (mgr.14397) 69 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi088.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T00:30:46.273 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[24895]: audit 2024-09-13T00:30:44.902808+0000 mgr.y (mgr.14397) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T00:30:46.273 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[24895]: audit 2024-09-13T00:30:44.910418+0000 mgr.y (mgr.14397) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi088.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T00:30:46.274 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[24895]: audit 2024-09-13T00:30:45.236901+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:46.274 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[24895]: audit 2024-09-13T00:30:45.242150+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:46.487 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[22845]: [13/Sep/2024:00:30:46] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:30:46.487 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[22845]: [13/Sep/2024:00:30:46] ENGINE Bus STOPPED 2024-09-13T00:30:46.487 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[22845]: [13/Sep/2024:00:30:46] ENGINE Bus STARTING 2024-09-13T00:30:46.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[32040]: audit 2024-09-13T00:30:44.830290+0000 mgr.y (mgr.14397) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T00:30:46.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[32040]: audit 2024-09-13T00:30:44.833208+0000 mgr.y (mgr.14397) 67 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi077.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T00:30:46.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[32040]: audit 2024-09-13T00:30:44.861070+0000 mgr.y (mgr.14397) 68 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:30:46.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[32040]: audit 2024-09-13T00:30:44.869205+0000 mgr.y (mgr.14397) 69 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi088.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T00:30:46.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[32040]: audit 2024-09-13T00:30:44.902808+0000 mgr.y (mgr.14397) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T00:30:46.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[32040]: audit 2024-09-13T00:30:44.910418+0000 mgr.y (mgr.14397) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi088.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T00:30:46.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[32040]: audit 2024-09-13T00:30:45.236901+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:46.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[32040]: audit 2024-09-13T00:30:45.242150+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:46.488 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[22596]: audit 2024-09-13T00:30:44.830290+0000 mgr.y (mgr.14397) 66 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T00:30:46.489 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[22596]: audit 2024-09-13T00:30:44.833208+0000 mgr.y (mgr.14397) 67 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi077.front.sepia.ceph.com:9093"}]: dispatch 2024-09-13T00:30:46.489 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[22596]: audit 2024-09-13T00:30:44.861070+0000 mgr.y (mgr.14397) 68 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:30:46.489 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[22596]: audit 2024-09-13T00:30:44.869205+0000 mgr.y (mgr.14397) 69 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi088.front.sepia.ceph.com:9095"}]: dispatch 2024-09-13T00:30:46.489 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[22596]: audit 2024-09-13T00:30:44.902808+0000 mgr.y (mgr.14397) 70 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T00:30:46.489 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[22596]: audit 2024-09-13T00:30:44.910418+0000 mgr.y (mgr.14397) 71 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi088.front.sepia.ceph.com:3000"}]: dispatch 2024-09-13T00:30:46.489 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[22596]: audit 2024-09-13T00:30:45.236901+0000 mon.a (mon.0) 720 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:46.489 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[22596]: audit 2024-09-13T00:30:45.242150+0000 mon.a (mon.0) 721 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:46.542 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.400Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T00:30:46.542 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.400Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T00:30:46.542 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.400Z caller=main.go:570 level=info host_details="(Linux 5.15.0-121-generic #131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024 x86_64 smithi088 (none))" 2024-09-13T00:30:46.542 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.400Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T00:30:46.542 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.401Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T00:30:46.543 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.402Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T00:30:46.543 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.403Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T00:30:46.543 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.403Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T00:30:46.543 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.403Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T00:30:46.543 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.406Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T00:30:46.543 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.406Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=12.551µs 2024-09-13T00:30:46.543 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.406Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T00:30:46.543 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.407Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-13T00:30:46.543 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.408Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-13T00:30:46.544 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.408Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=36.774µs wal_replay_duration=1.771065ms wbl_replay_duration=97ns total_replay_duration=1.864629ms 2024-09-13T00:30:46.544 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.409Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T00:30:46.544 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.409Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T00:30:46.544 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.409Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T00:30:46.544 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.425Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=16.557539ms db_storage=855ns remote_storage=1.476µs web_handler=391ns query_engine=627ns scrape=314.938µs scrape_sd=29.172µs notify=22.54µs notify_sd=7.049µs rules=15.885766ms tracing=7.981µs 2024-09-13T00:30:46.544 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.425Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T00:30:46.544 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51748]: ts=2024-09-13T00:30:46.425Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T00:30:46.544 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.276535663Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-13T00:30:46.544 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.318560541Z level=info msg="Executing migration" id="add correlation config column" 2024-09-13T00:30:46.545 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.360834142Z level=info msg="Executing migration" id="create entity_events table" 2024-09-13T00:30:46.545 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.402894404Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-13T00:30:46.545 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.470737921Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-13T00:30:46.545 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.470929514Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-13T00:30:46.545 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.495899611Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T00:30:46.545 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.49602533Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T00:30:46.545 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.521079323Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-13T00:30:46.545 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.546344457Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-13T00:30:46.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[22845]: [13/Sep/2024:00:30:46] ENGINE Serving on http://:::9283 2024-09-13T00:30:46.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:46 smithi077 bash[22845]: [13/Sep/2024:00:30:46] ENGINE Bus STARTED 2024-09-13T00:30:46.803 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.571553034Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-13T00:30:46.803 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.61369425Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-13T00:30:46.803 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.638951477Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-13T00:30:46.803 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.680899647Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-13T00:30:46.803 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.70619183Z level=info msg="Executing migration" id="Drop public config table" 2024-09-13T00:30:46.803 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.731435097Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-13T00:30:46.803 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.773401534Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-13T00:30:46.803 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.790299721Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-13T00:30:46.803 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.807222352Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-13T00:30:47.072 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.824156452Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-13T00:30:47.072 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.849304307Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-13T00:30:47.072 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.907979803Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-13T00:30:47.072 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.92488344Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-13T00:30:47.073 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.950050348Z level=info msg="Executing migration" id="add share column" 2024-09-13T00:30:47.073 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:46 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:46.975371274Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-13T00:30:47.073 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.000599352Z level=info msg="Executing migration" id="create file table" 2024-09-13T00:30:47.073 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.025919065Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-13T00:30:47.073 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.051117386Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-13T00:30:47.073 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.076394496Z level=info msg="Executing migration" id="create file_meta table" 2024-09-13T00:30:47.341 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[24895]: cluster 2024-09-13T00:30:45.772033+0000 mgr.y (mgr.14397) 72 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:47.341 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.101576804Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-13T00:30:47.342 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.126802991Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-13T00:30:47.342 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.168699698Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-13T00:30:47.342 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.210694372Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-13T00:30:47.342 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.252640251Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-13T00:30:47.342 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.29471298Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-13T00:30:47.342 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.320010814Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-13T00:30:47.342 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.345574408Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-13T00:30:47.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:47 smithi077 bash[22596]: cluster 2024-09-13T00:30:45.772033+0000 mgr.y (mgr.14397) 72 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:47.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:47 smithi077 bash[32040]: cluster 2024-09-13T00:30:45.772033+0000 mgr.y (mgr.14397) 72 : cluster [DBG] pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:47.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.370889261Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-13T00:30:47.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.395984763Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-13T00:30:47.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.42126419Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-13T00:30:47.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.446366369Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-13T00:30:47.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.4885427Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-13T00:30:47.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.513653895Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-13T00:30:47.600 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.555670935Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-13T00:30:47.890 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:30:47.921 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.639373456Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-13T00:30:47.921 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.681452232Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-13T00:30:47.921 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.706710629Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-13T00:30:47.921 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.748632583Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-13T00:30:47.921 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.790428784Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-13T00:30:47.921 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.815756467Z level=info msg="Executing migration" id="create folder table" 2024-09-13T00:30:47.921 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.85773251Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-13T00:30:47.922 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.883060657Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-13T00:30:47.922 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.924944451Z level=info msg="Executing migration" id="Update folder title length" 2024-09-13T00:30:48.350 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.966869025Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-13T00:30:48.350 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=migrator t=2024-09-13T00:30:47.991883346Z level=info msg="migrations completed" performed=485 skipped=0 duration=20.721752431s 2024-09-13T00:30:48.350 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:47 smithi088 bash[51081]: logger=sqlstore t=2024-09-13T00:30:47.992754279Z level=info msg="Created default organization" 2024-09-13T00:30:48.350 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:48 smithi088 bash[51081]: logger=plugin.loader t=2024-09-13T00:30:48.079939913Z level=info msg="Plugin registered" pluginID=input 2024-09-13T00:30:48.350 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:48 smithi088 bash[51081]: logger=plugin.loader t=2024-09-13T00:30:48.108763431Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-13T00:30:48.350 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:48 smithi088 bash[51081]: logger=plugin.loader t=2024-09-13T00:30:48.108786736Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-13T00:30:48.351 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:48 smithi088 bash[51081]: logger=secrets t=2024-09-13T00:30:48.108823886Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-13T00:30:48.351 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:48 smithi088 bash[51081]: logger=query_data t=2024-09-13T00:30:48.110845578Z level=info msg="Query Service initialization" 2024-09-13T00:30:48.351 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:48 smithi088 bash[51081]: logger=live.push_http t=2024-09-13T00:30:48.114073717Z level=info msg="Live Push Gateway initialization" 2024-09-13T00:30:49.523 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:49 smithi088 bash[24895]: cluster 2024-09-13T00:30:47.772603+0000 mgr.y (mgr.14397) 73 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:49.523 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:49 smithi088 bash[51081]: logger=infra.usagestats.collector t=2024-09-13T00:30:49.333597807Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-13T00:30:49.524 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:49 smithi088 bash[51081]: logger=provisioning.datasources t=2024-09-13T00:30:49.334466259Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-09-13T00:30:49.524 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:49 smithi088 bash[51081]: logger=provisioning.datasources t=2024-09-13T00:30:49.367775989Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T00:30:49.524 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:49 smithi088 bash[51081]: logger=provisioning.datasources t=2024-09-13T00:30:49.462045374Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-13T00:30:49.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:49 smithi077 bash[22596]: cluster 2024-09-13T00:30:47.772603+0000 mgr.y (mgr.14397) 73 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:49.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:49 smithi077 bash[32040]: cluster 2024-09-13T00:30:47.772603+0000 mgr.y (mgr.14397) 73 : cluster [DBG] pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:49.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:49 smithi088 bash[51081]: logger=provisioning.alerting t=2024-09-13T00:30:49.52772288Z level=info msg="starting to provision alerting" 2024-09-13T00:30:49.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:49 smithi088 bash[51081]: logger=provisioning.alerting t=2024-09-13T00:30:49.527786413Z level=info msg="finished to provision alerting" 2024-09-13T00:30:49.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:49 smithi088 bash[51081]: logger=ngalert.state.manager t=2024-09-13T00:30:49.528030323Z level=info msg="Warming state cache for startup" 2024-09-13T00:30:49.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:49 smithi088 bash[51081]: logger=grafanaStorageLogger t=2024-09-13T00:30:49.52860608Z level=info msg="storage starting" 2024-09-13T00:30:49.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:49 smithi088 bash[51081]: logger=http.server t=2024-09-13T00:30:49.532745318Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T00:30:49.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:49 smithi088 bash[51081]: logger=sqlstore.transactions t=2024-09-13T00:30:49.53953696Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T00:30:49.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:49 smithi088 bash[51081]: logger=ngalert.state.manager t=2024-09-13T00:30:49.610586899Z level=info msg="State cache has been initialized" states=0 duration=82.554632ms 2024-09-13T00:30:49.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:49 smithi088 bash[51081]: logger=ngalert.multiorg.alertmanager t=2024-09-13T00:30:49.610660676Z level=info msg="starting MultiOrg Alertmanager" 2024-09-13T00:30:49.851 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:30:49 smithi088 bash[51081]: logger=ticker t=2024-09-13T00:30:49.610688349Z level=info msg=starting first_tick=2024-09-13T00:30:50Z 2024-09-13T00:30:50.276 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:30:50 smithi077 bash[58266]: ts=2024-09-13T00:30:50.152Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002964112s 2024-09-13T00:30:50.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:50 smithi077 bash[22596]: audit 2024-09-13T00:30:49.450581+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:50.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:50 smithi077 bash[22596]: audit 2024-09-13T00:30:49.457835+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:50.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:50 smithi077 bash[32040]: audit 2024-09-13T00:30:49.450581+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:50.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:50 smithi077 bash[32040]: audit 2024-09-13T00:30:49.457835+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:50.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:50 smithi088 bash[24895]: audit 2024-09-13T00:30:49.450581+0000 mon.a (mon.0) 722 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:50.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:50 smithi088 bash[24895]: audit 2024-09-13T00:30:49.457835+0000 mon.a (mon.0) 723 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:51.692 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:30:51.692 INFO:teuthology.orchestra.run.smithi077.stdout:{"epoch":58,"fsid":"9218306c-7165-11ef-bceb-c7b262605968","created":"2024-09-13T00:18:59.375829+0000","modified":"2024-09-13T00:30:30.837095+0000","last_up_change":"2024-09-13T00:28:29.926728+0000","last_in_change":"2024-09-13T00:28:05.097746+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T00:24:15.726444+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T00:30:09.880408+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":57,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"e0860feb-72c9-4c49-933d-0a49b68caeee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6802","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6803","nonce":1623816886}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6804","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6805","nonce":1623816886}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6808","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6809","nonce":1623816886}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6806","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6807","nonce":1623816886}]},"public_addr":"172.21.15.77:6803/1623816886","cluster_addr":"172.21.15.77:6805/1623816886","heartbeat_back_addr":"172.21.15.77:6809/1623816886","heartbeat_front_addr":"172.21.15.77:6807/1623816886","state":["exists","up"]},{"osd":1,"uuid":"7ee01388-61b0-4a60-81f2-08c3c072ec0e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6810","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6811","nonce":1691613782}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6812","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6813","nonce":1691613782}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6816","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6817","nonce":1691613782}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6814","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6815","nonce":1691613782}]},"public_addr":"172.21.15.77:6811/1691613782","cluster_addr":"172.21.15.77:6813/1691613782","heartbeat_back_addr":"172.21.15.77:6817/1691613782","heartbeat_front_addr":"172.21.15.77:6815/1691613782","state":["exists","up"]},{"osd":2,"uuid":"ff8bc88a-5b11-469f-912d-d3149036b450","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6818","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6819","nonce":2379210664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6820","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6821","nonce":2379210664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6824","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6825","nonce":2379210664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6822","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6823","nonce":2379210664}]},"public_addr":"172.21.15.77:6819/2379210664","cluster_addr":"172.21.15.77:6821/2379210664","heartbeat_back_addr":"172.21.15.77:6825/2379210664","heartbeat_front_addr":"172.21.15.77:6823/2379210664","state":["exists","up"]},{"osd":3,"uuid":"cb032137-609e-41f6-bfc7-e70b77ea9456","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6826","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6827","nonce":2636296765}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6828","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6829","nonce":2636296765}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6832","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6833","nonce":2636296765}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6830","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6831","nonce":2636296765}]},"public_addr":"172.21.15.77:6827/2636296765","cluster_addr":"172.21.15.77:6829/2636296765","heartbeat_back_addr":"172.21.15.77:6833/2636296765","heartbeat_front_addr":"172.21.15.77:6831/2636296765","state":["exists","up"]},{"osd":4,"uuid":"cd38603e-5cdc-42b6-b8c8-656d44df0ed5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6800","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6801","nonce":3645723145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6802","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6803","nonce":3645723145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6806","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6807","nonce":3645723145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6804","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6805","nonce":3645723145}]},"public_addr":"172.21.15.88:6801/3645723145","cluster_addr":"172.21.15.88:6803/3645723145","heartbeat_back_addr":"172.21.15.88:6807/3645723145","heartbeat_front_addr":"172.21.15.88:6805/3645723145","state":["exists","up"]},{"osd":5,"uuid":"d4b81ff3-f257-4f63-a7a7-cbd1ff754544","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6808","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6809","nonce":2699666173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6810","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6811","nonce":2699666173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6814","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6815","nonce":2699666173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6812","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6813","nonce":2699666173}]},"public_addr":"172.21.15.88:6809/2699666173","cluster_addr":"172.21.15.88:6811/2699666173","heartbeat_back_addr":"172.21.15.88:6815/2699666173","heartbeat_front_addr":"172.21.15.88:6813/2699666173","state":["exists","up"]},{"osd":6,"uuid":"31c0d169-c329-440d-848b-057fbbf418ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6816","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6817","nonce":412460705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6818","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6819","nonce":412460705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6822","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6823","nonce":412460705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6820","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6821","nonce":412460705}]},"public_addr":"172.21.15.88:6817/412460705","cluster_addr":"172.21.15.88:6819/412460705","heartbeat_back_addr":"172.21.15.88:6823/412460705","heartbeat_front_addr":"172.21.15.88:6821/412460705","state":["exists","up"]},{"osd":7,"uuid":"2b2c1a7c-3203-4e9a-b07b-4eaae7e09c0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6824","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6825","nonce":2009347492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6826","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6827","nonce":2009347492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6830","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6831","nonce":2009347492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6828","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6829","nonce":2009347492}]},"public_addr":"172.21.15.88:6825/2009347492","cluster_addr":"172.21.15.88:6827/2009347492","heartbeat_back_addr":"172.21.15.88:6831/2009347492","heartbeat_front_addr":"172.21.15.88:6829/2009347492","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:22:21.643909+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:23:17.132405+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:24:13.726010+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:25:11.211228+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:26:00.949032+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:26:49.596019+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:27:38.694576+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:28:28.383054+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.77:0/1008984793":"2024-09-14T00:29:27.728349+0000","172.21.15.77:6801/395417143":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/892433806":"2024-09-14T00:19:25.064947+0000","172.21.15.77:0/4291417054":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6800/426595304":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6801/426595304":"2024-09-14T00:19:25.064947+0000","172.21.15.77:0/1137737372":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6800/395417143":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/1897411349":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/4117010497":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/2198980039":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/1347196058":"2024-09-14T00:29:27.728349+0000","172.21.15.77:6800/3213372405":"2024-09-14T00:19:55.144762+0000","172.21.15.77:6801/3213372405":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/1177255174":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/3827884320":"2024-09-14T00:29:27.728349+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T00:30:51.704 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:30:51] "GET /metrics HTTP/1.1" 200 203969 "" "Prometheus/2.43.0" 2024-09-13T00:30:51.704 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[32040]: cluster 2024-09-13T00:30:49.773180+0000 mgr.y (mgr.14397) 74 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:51.704 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[32040]: audit 2024-09-13T00:30:50.812384+0000 mon.a (mon.0) 724 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:51.704 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[32040]: audit 2024-09-13T00:30:50.825650+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:51.704 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[32040]: audit 2024-09-13T00:30:51.220351+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:30:51.704 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[32040]: audit 2024-09-13T00:30:51.222231+0000 mon.c (mon.2) 61 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:30:51.704 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[32040]: audit 2024-09-13T00:30:51.235146+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:51.704 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[32040]: audit 2024-09-13T00:30:51.279397+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:30:51.705 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[32040]: audit 2024-09-13T00:30:51.282654+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:30:51.705 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[32040]: audit 2024-09-13T00:30:51.284161+0000 mon.c (mon.2) 64 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:30:51.705 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[32040]: audit 2024-09-13T00:30:51.292441+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:51.705 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[22596]: cluster 2024-09-13T00:30:49.773180+0000 mgr.y (mgr.14397) 74 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:51.705 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[22596]: audit 2024-09-13T00:30:50.812384+0000 mon.a (mon.0) 724 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:51.705 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[22596]: audit 2024-09-13T00:30:50.825650+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:51.705 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[22596]: audit 2024-09-13T00:30:51.220351+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:30:51.705 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[22596]: audit 2024-09-13T00:30:51.222231+0000 mon.c (mon.2) 61 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:30:51.705 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[22596]: audit 2024-09-13T00:30:51.235146+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:51.705 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[22596]: audit 2024-09-13T00:30:51.279397+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:30:51.706 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[22596]: audit 2024-09-13T00:30:51.282654+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:30:51.706 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[22596]: audit 2024-09-13T00:30:51.284161+0000 mon.c (mon.2) 64 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:30:51.706 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:51 smithi077 bash[22596]: audit 2024-09-13T00:30:51.292441+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:51.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:51 smithi088 bash[24895]: cluster 2024-09-13T00:30:49.773180+0000 mgr.y (mgr.14397) 74 : cluster [DBG] pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:51.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:51 smithi088 bash[24895]: audit 2024-09-13T00:30:50.812384+0000 mon.a (mon.0) 724 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:51.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:51 smithi088 bash[24895]: audit 2024-09-13T00:30:50.825650+0000 mon.a (mon.0) 725 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:51.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:51 smithi088 bash[24895]: audit 2024-09-13T00:30:51.220351+0000 mon.c (mon.2) 60 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:30:51.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:51 smithi088 bash[24895]: audit 2024-09-13T00:30:51.222231+0000 mon.c (mon.2) 61 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:30:51.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:51 smithi088 bash[24895]: audit 2024-09-13T00:30:51.235146+0000 mon.a (mon.0) 726 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:51.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:51 smithi088 bash[24895]: audit 2024-09-13T00:30:51.279397+0000 mon.c (mon.2) 62 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:30:51.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:51 smithi088 bash[24895]: audit 2024-09-13T00:30:51.282654+0000 mon.c (mon.2) 63 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:30:51.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:51 smithi088 bash[24895]: audit 2024-09-13T00:30:51.284161+0000 mon.c (mon.2) 64 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:30:51.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:51 smithi088 bash[24895]: audit 2024-09-13T00:30:51.292441+0000 mon.a (mon.0) 727 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:30:52.452 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T00:30:52.452 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T00:30:52.453 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T00:30:52.453 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T00:30:52.454 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T00:30:52.454 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T00:30:52.454 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T00:30:52.455 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T00:30:52.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:52 smithi077 bash[22596]: audit 2024-09-13T00:30:51.695555+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.77:0/3973518262' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:30:52.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:52 smithi077 bash[32040]: audit 2024-09-13T00:30:51.695555+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.77:0/3973518262' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:30:52.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:52 smithi088 bash[24895]: audit 2024-09-13T00:30:51.695555+0000 mon.c (mon.2) 65 : audit [DBG] from='client.? 172.21.15.77:0/3973518262' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:30:53.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:53 smithi077 bash[32040]: cluster 2024-09-13T00:30:51.773669+0000 mgr.y (mgr.14397) 75 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:53.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:53 smithi077 bash[22596]: cluster 2024-09-13T00:30:51.773669+0000 mgr.y (mgr.14397) 75 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:53.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:53 smithi088 bash[24895]: cluster 2024-09-13T00:30:51.773669+0000 mgr.y (mgr.14397) 75 : cluster [DBG] pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:55.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:55 smithi077 bash[22596]: cluster 2024-09-13T00:30:53.774264+0000 mgr.y (mgr.14397) 76 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:55.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:55 smithi077 bash[32040]: cluster 2024-09-13T00:30:53.774264+0000 mgr.y (mgr.14397) 76 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:55.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:55 smithi088 bash[24895]: cluster 2024-09-13T00:30:53.774264+0000 mgr.y (mgr.14397) 76 : cluster [DBG] pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:57.355 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:30:57.356 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:30:57.356 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:30:57.357 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:30:57.358 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:30:57.358 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:30:57.360 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:30:57.362 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:30:57.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:57 smithi077 bash[22596]: cluster 2024-09-13T00:30:55.775009+0000 mgr.y (mgr.14397) 77 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:57.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:57 smithi077 bash[32040]: cluster 2024-09-13T00:30:55.775009+0000 mgr.y (mgr.14397) 77 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:57.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:57 smithi088 bash[24895]: cluster 2024-09-13T00:30:55.775009+0000 mgr.y (mgr.14397) 77 : cluster [DBG] pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:58.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:58 smithi077 bash[22596]: audit 2024-09-13T00:30:57.827497+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:58.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:58 smithi077 bash[32040]: audit 2024-09-13T00:30:57.827497+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:58.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:58 smithi088 bash[24895]: audit 2024-09-13T00:30:57.827497+0000 mon.c (mon.2) 66 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:30:59.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:30:59 smithi077 bash[22596]: cluster 2024-09-13T00:30:57.775665+0000 mgr.y (mgr.14397) 78 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:59.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:30:59 smithi077 bash[32040]: cluster 2024-09-13T00:30:57.775665+0000 mgr.y (mgr.14397) 78 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:30:59.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:30:59 smithi088 bash[24895]: cluster 2024-09-13T00:30:57.775665+0000 mgr.y (mgr.14397) 78 : cluster [DBG] pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:00.718 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:00 smithi077 bash[22596]: cluster 2024-09-13T00:30:59.776358+0000 mgr.y (mgr.14397) 79 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:00.719 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:00 smithi077 bash[32040]: cluster 2024-09-13T00:30:59.776358+0000 mgr.y (mgr.14397) 79 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:00.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:00 smithi088 bash[24895]: cluster 2024-09-13T00:30:59.776358+0000 mgr.y (mgr.14397) 79 : cluster [DBG] pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:00.850 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:31:00 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:31:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:31:01.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:31:01 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:31:01] "GET /metrics HTTP/1.1" 200 203959 "" "Prometheus/2.43.0" 2024-09-13T00:31:03.099 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:02 smithi088 bash[24895]: cluster 2024-09-13T00:31:01.776921+0000 mgr.y (mgr.14397) 80 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:03.100 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:02 smithi077 bash[22596]: cluster 2024-09-13T00:31:01.776921+0000 mgr.y (mgr.14397) 80 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:03.101 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:02 smithi077 bash[32040]: cluster 2024-09-13T00:31:01.776921+0000 mgr.y (mgr.14397) 80 : cluster [DBG] pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:05.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:04 smithi088 bash[24895]: cluster 2024-09-13T00:31:03.777479+0000 mgr.y (mgr.14397) 81 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:05.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:04 smithi077 bash[22596]: cluster 2024-09-13T00:31:03.777479+0000 mgr.y (mgr.14397) 81 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:05.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:05 smithi077 bash[32040]: cluster 2024-09-13T00:31:03.777479+0000 mgr.y (mgr.14397) 81 : cluster [DBG] pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:07.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:06 smithi077 bash[22596]: cluster 2024-09-13T00:31:05.778115+0000 mgr.y (mgr.14397) 82 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:07.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:06 smithi077 bash[32040]: cluster 2024-09-13T00:31:05.778115+0000 mgr.y (mgr.14397) 82 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:07.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:06 smithi088 bash[24895]: cluster 2024-09-13T00:31:05.778115+0000 mgr.y (mgr.14397) 82 : cluster [DBG] pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:09.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:08 smithi077 bash[22596]: cluster 2024-09-13T00:31:07.778703+0000 mgr.y (mgr.14397) 83 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:09.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:08 smithi077 bash[32040]: cluster 2024-09-13T00:31:07.778703+0000 mgr.y (mgr.14397) 83 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:09.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:08 smithi088 bash[24895]: cluster 2024-09-13T00:31:07.778703+0000 mgr.y (mgr.14397) 83 : cluster [DBG] pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:11.099 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:10 smithi088 bash[24895]: cluster 2024-09-13T00:31:09.779328+0000 mgr.y (mgr.14397) 84 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:11.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:31:10 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:31:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:31:11.225 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:10 smithi077 bash[22596]: cluster 2024-09-13T00:31:09.779328+0000 mgr.y (mgr.14397) 84 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:11.226 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:10 smithi077 bash[32040]: cluster 2024-09-13T00:31:09.779328+0000 mgr.y (mgr.14397) 84 : cluster [DBG] pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:11.501 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:31:11 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:31:11] "GET /metrics HTTP/1.1" 200 203959 "" "Prometheus/2.43.0" 2024-09-13T00:31:13.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:12 smithi077 bash[32040]: cluster 2024-09-13T00:31:11.779906+0000 mgr.y (mgr.14397) 85 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:13.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:12 smithi077 bash[32040]: audit 2024-09-13T00:31:12.828066+0000 mon.c (mon.2) 67 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:13.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:12 smithi077 bash[22596]: cluster 2024-09-13T00:31:11.779906+0000 mgr.y (mgr.14397) 85 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:13.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:12 smithi077 bash[22596]: audit 2024-09-13T00:31:12.828066+0000 mon.c (mon.2) 67 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:13.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:12 smithi088 bash[24895]: cluster 2024-09-13T00:31:11.779906+0000 mgr.y (mgr.14397) 85 : cluster [DBG] pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:13.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:12 smithi088 bash[24895]: audit 2024-09-13T00:31:12.828066+0000 mon.c (mon.2) 67 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:13.679 INFO:teuthology.orchestra.run.smithi077.stdout:77309411413 2024-09-13T00:31:13.680 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T00:31:14.523 INFO:teuthology.orchestra.run.smithi077.stdout:128849018943 2024-09-13T00:31:14.523 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T00:31:15.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:14 smithi077 bash[22596]: cluster 2024-09-13T00:31:13.780329+0000 mgr.y (mgr.14397) 86 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:15.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:14 smithi077 bash[32040]: cluster 2024-09-13T00:31:13.780329+0000 mgr.y (mgr.14397) 86 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:15.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:14 smithi088 bash[24895]: cluster 2024-09-13T00:31:13.780329+0000 mgr.y (mgr.14397) 86 : cluster [DBG] pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:15.720 INFO:teuthology.orchestra.run.smithi077.stdout:107374182473 2024-09-13T00:31:15.721 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T00:31:16.509 INFO:teuthology.orchestra.run.smithi077.stdout:180388626476 2024-09-13T00:31:16.510 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T00:31:16.595 INFO:teuthology.orchestra.run.smithi077.stdout:154618822710 2024-09-13T00:31:16.595 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T00:31:17.100 INFO:teuthology.orchestra.run.smithi077.stdout:34359738475 2024-09-13T00:31:17.101 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T00:31:17.114 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:16 smithi077 bash[32040]: cluster 2024-09-13T00:31:15.780739+0000 mgr.y (mgr.14397) 87 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:17.115 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:16 smithi077 bash[22596]: cluster 2024-09-13T00:31:15.780739+0000 mgr.y (mgr.14397) 87 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:17.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:16 smithi088 bash[24895]: cluster 2024-09-13T00:31:15.780739+0000 mgr.y (mgr.14397) 87 : cluster [DBG] pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:18.371 INFO:teuthology.orchestra.run.smithi077.stdout:55834574946 2024-09-13T00:31:18.371 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T00:31:18.388 INFO:teuthology.orchestra.run.smithi077.stdout:206158430243 2024-09-13T00:31:18.389 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T00:31:19.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:18 smithi077 bash[22596]: cluster 2024-09-13T00:31:17.781315+0000 mgr.y (mgr.14397) 88 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:19.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:18 smithi077 bash[32040]: cluster 2024-09-13T00:31:17.781315+0000 mgr.y (mgr.14397) 88 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:19.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:18 smithi088 bash[24895]: cluster 2024-09-13T00:31:17.781315+0000 mgr.y (mgr.14397) 88 : cluster [DBG] pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:20.885 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:31:20.885 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:31:20.888 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:31:20.890 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:31:20.891 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:31:21.099 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:20 smithi088 bash[24895]: cluster 2024-09-13T00:31:19.781968+0000 mgr.y (mgr.14397) 89 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:21.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:31:20 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:31:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:31:21.168 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:20 smithi077 bash[32040]: cluster 2024-09-13T00:31:19.781968+0000 mgr.y (mgr.14397) 89 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:21.168 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:20 smithi077 bash[22596]: cluster 2024-09-13T00:31:19.781968+0000 mgr.y (mgr.14397) 89 : cluster [DBG] pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:21.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:31:21 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:31:21] "GET /metrics HTTP/1.1" 200 203928 "" "Prometheus/2.43.0" 2024-09-13T00:31:21.906 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:31:22.926 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:31:22.926 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:31:23.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:22 smithi077 bash[22596]: cluster 2024-09-13T00:31:21.782576+0000 mgr.y (mgr.14397) 90 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:23.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:22 smithi077 bash[32040]: cluster 2024-09-13T00:31:21.782576+0000 mgr.y (mgr.14397) 90 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:23.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:22 smithi088 bash[24895]: cluster 2024-09-13T00:31:21.782576+0000 mgr.y (mgr.14397) 90 : cluster [DBG] pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:25.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:24 smithi077 bash[22596]: cluster 2024-09-13T00:31:23.783212+0000 mgr.y (mgr.14397) 91 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:25.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:24 smithi077 bash[32040]: cluster 2024-09-13T00:31:23.783212+0000 mgr.y (mgr.14397) 91 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:25.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:24 smithi088 bash[24895]: cluster 2024-09-13T00:31:23.783212+0000 mgr.y (mgr.14397) 91 : cluster [DBG] pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:27.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:26 smithi077 bash[22596]: cluster 2024-09-13T00:31:25.783766+0000 mgr.y (mgr.14397) 92 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:27.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:26 smithi077 bash[32040]: cluster 2024-09-13T00:31:25.783766+0000 mgr.y (mgr.14397) 92 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:27.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:26 smithi088 bash[24895]: cluster 2024-09-13T00:31:25.783766+0000 mgr.y (mgr.14397) 92 : cluster [DBG] pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:28.291 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:27 smithi077 bash[22596]: audit 2024-09-13T00:31:27.830651+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:28.291 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:27 smithi077 bash[32040]: audit 2024-09-13T00:31:27.830651+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:28.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:27 smithi088 bash[24895]: audit 2024-09-13T00:31:27.830651+0000 mon.c (mon.2) 68 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:29.171 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:28 smithi077 bash[22596]: cluster 2024-09-13T00:31:27.784276+0000 mgr.y (mgr.14397) 93 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:29.172 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:28 smithi077 bash[32040]: cluster 2024-09-13T00:31:27.784276+0000 mgr.y (mgr.14397) 93 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:29.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:28 smithi088 bash[24895]: cluster 2024-09-13T00:31:27.784276+0000 mgr.y (mgr.14397) 93 : cluster [DBG] pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:31.099 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:30 smithi088 bash[24895]: cluster 2024-09-13T00:31:29.784880+0000 mgr.y (mgr.14397) 94 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:31.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:31:30 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:31:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:31:31.297 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:30 smithi077 bash[32040]: cluster 2024-09-13T00:31:29.784880+0000 mgr.y (mgr.14397) 94 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:31.297 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:30 smithi077 bash[22596]: cluster 2024-09-13T00:31:29.784880+0000 mgr.y (mgr.14397) 94 : cluster [DBG] pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:31.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:31:31 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:31:31] "GET /metrics HTTP/1.1" 200 203927 "" "Prometheus/2.43.0" 2024-09-13T00:31:33.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:32 smithi077 bash[22596]: cluster 2024-09-13T00:31:31.785407+0000 mgr.y (mgr.14397) 95 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:33.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:32 smithi077 bash[32040]: cluster 2024-09-13T00:31:31.785407+0000 mgr.y (mgr.14397) 95 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:33.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:32 smithi088 bash[24895]: cluster 2024-09-13T00:31:31.785407+0000 mgr.y (mgr.14397) 95 : cluster [DBG] pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:35.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:34 smithi077 bash[22596]: cluster 2024-09-13T00:31:33.785963+0000 mgr.y (mgr.14397) 96 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:35.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:34 smithi077 bash[32040]: cluster 2024-09-13T00:31:33.785963+0000 mgr.y (mgr.14397) 96 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:35.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:34 smithi088 bash[24895]: cluster 2024-09-13T00:31:33.785963+0000 mgr.y (mgr.14397) 96 : cluster [DBG] pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:35.418 INFO:teuthology.orchestra.run.smithi077.stdout:77309411417 2024-09-13T00:31:35.996 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:35 smithi077 bash[22596]: audit 2024-09-13T00:31:35.424060+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.77:0/3190230467' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T00:31:36.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:36 smithi077 bash[32040]: audit 2024-09-13T00:31:35.424060+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.77:0/3190230467' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T00:31:36.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:35 smithi088 bash[24895]: audit 2024-09-13T00:31:35.424060+0000 mon.c (mon.2) 69 : audit [DBG] from='client.? 172.21.15.77:0/3190230467' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T00:31:37.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:36 smithi077 bash[22596]: cluster 2024-09-13T00:31:35.786603+0000 mgr.y (mgr.14397) 97 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:37.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:36 smithi077 bash[32040]: cluster 2024-09-13T00:31:35.786603+0000 mgr.y (mgr.14397) 97 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:37.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:36 smithi088 bash[24895]: cluster 2024-09-13T00:31:35.786603+0000 mgr.y (mgr.14397) 97 : cluster [DBG] pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:37.965 INFO:teuthology.orchestra.run.smithi077.stdout:154618822715 2024-09-13T00:31:38.233 INFO:teuthology.orchestra.run.smithi077.stdout:128849018948 2024-09-13T00:31:39.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:38 smithi077 bash[22596]: cluster 2024-09-13T00:31:37.786979+0000 mgr.y (mgr.14397) 98 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:39.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:38 smithi077 bash[22596]: audit 2024-09-13T00:31:37.970562+0000 mon.a (mon.0) 728 : audit [DBG] from='client.? 172.21.15.77:0/3277156530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T00:31:39.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:38 smithi077 bash[22596]: audit 2024-09-13T00:31:38.239010+0000 mon.a (mon.0) 729 : audit [DBG] from='client.? 172.21.15.77:0/2745498522' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T00:31:39.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:38 smithi077 bash[32040]: cluster 2024-09-13T00:31:37.786979+0000 mgr.y (mgr.14397) 98 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:39.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:38 smithi077 bash[32040]: audit 2024-09-13T00:31:37.970562+0000 mon.a (mon.0) 728 : audit [DBG] from='client.? 172.21.15.77:0/3277156530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T00:31:39.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:38 smithi077 bash[32040]: audit 2024-09-13T00:31:38.239010+0000 mon.a (mon.0) 729 : audit [DBG] from='client.? 172.21.15.77:0/2745498522' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T00:31:39.043 INFO:teuthology.orchestra.run.smithi077.stdout:107374182478 2024-09-13T00:31:39.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:38 smithi088 bash[24895]: cluster 2024-09-13T00:31:37.786979+0000 mgr.y (mgr.14397) 98 : cluster [DBG] pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:39.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:38 smithi088 bash[24895]: audit 2024-09-13T00:31:37.970562+0000 mon.a (mon.0) 728 : audit [DBG] from='client.? 172.21.15.77:0/3277156530' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T00:31:39.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:38 smithi088 bash[24895]: audit 2024-09-13T00:31:38.239010+0000 mon.a (mon.0) 729 : audit [DBG] from='client.? 172.21.15.77:0/2745498522' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T00:31:39.372 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411413 got 77309411417 for osd.2 2024-09-13T00:31:39.372 DEBUG:teuthology.parallel:result is None 2024-09-13T00:31:39.794 INFO:teuthology.orchestra.run.smithi077.stdout:34359738479 2024-09-13T00:31:40.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:39 smithi077 bash[32040]: audit 2024-09-13T00:31:39.049279+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.77:0/3089712768' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T00:31:40.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:39 smithi077 bash[32040]: audit 2024-09-13T00:31:39.800071+0000 mon.a (mon.0) 730 : audit [DBG] from='client.? 172.21.15.77:0/1949878644' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T00:31:40.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:39 smithi077 bash[22596]: audit 2024-09-13T00:31:39.049279+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.77:0/3089712768' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T00:31:40.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:39 smithi077 bash[22596]: audit 2024-09-13T00:31:39.800071+0000 mon.a (mon.0) 730 : audit [DBG] from='client.? 172.21.15.77:0/1949878644' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T00:31:40.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:39 smithi088 bash[24895]: audit 2024-09-13T00:31:39.049279+0000 mon.c (mon.2) 70 : audit [DBG] from='client.? 172.21.15.77:0/3089712768' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T00:31:40.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:39 smithi088 bash[24895]: audit 2024-09-13T00:31:39.800071+0000 mon.a (mon.0) 730 : audit [DBG] from='client.? 172.21.15.77:0/1949878644' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T00:31:41.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:40 smithi077 bash[22596]: cluster 2024-09-13T00:31:39.787549+0000 mgr.y (mgr.14397) 99 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:41.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:40 smithi077 bash[32040]: cluster 2024-09-13T00:31:39.787549+0000 mgr.y (mgr.14397) 99 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:41.033 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822710 got 154618822715 for osd.5 2024-09-13T00:31:41.033 DEBUG:teuthology.parallel:result is None 2024-09-13T00:31:41.099 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:40 smithi088 bash[24895]: cluster 2024-09-13T00:31:39.787549+0000 mgr.y (mgr.14397) 99 : cluster [DBG] pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:41.099 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:31:40 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:31:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:31:41.508 INFO:teuthology.orchestra.run.smithi077.stdout:180388626482 2024-09-13T00:31:41.631 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018943 got 128849018948 for osd.4 2024-09-13T00:31:41.631 DEBUG:teuthology.parallel:result is None 2024-09-13T00:31:41.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:31:41 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:31:41] "GET /metrics HTTP/1.1" 200 203927 "" "Prometheus/2.43.0" 2024-09-13T00:31:42.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:41 smithi077 bash[22596]: audit 2024-09-13T00:31:41.513806+0000 mon.a (mon.0) 731 : audit [DBG] from='client.? 172.21.15.77:0/3559541955' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T00:31:42.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:42 smithi077 bash[32040]: audit 2024-09-13T00:31:41.513806+0000 mon.a (mon.0) 731 : audit [DBG] from='client.? 172.21.15.77:0/3559541955' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T00:31:42.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:41 smithi088 bash[24895]: audit 2024-09-13T00:31:41.513806+0000 mon.a (mon.0) 731 : audit [DBG] from='client.? 172.21.15.77:0/3559541955' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T00:31:43.007 INFO:teuthology.orchestra.run.smithi077.stdout:206158430248 2024-09-13T00:31:43.097 INFO:teuthology.orchestra.run.smithi077.stdout:55834574951 2024-09-13T00:31:43.220 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:42 smithi077 bash[32040]: cluster 2024-09-13T00:31:41.788045+0000 mgr.y (mgr.14397) 100 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:43.220 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:42 smithi077 bash[32040]: audit 2024-09-13T00:31:42.830197+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:43.221 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:42 smithi077 bash[22596]: cluster 2024-09-13T00:31:41.788045+0000 mgr.y (mgr.14397) 100 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:43.221 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:42 smithi077 bash[22596]: audit 2024-09-13T00:31:42.830197+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:43.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:42 smithi088 bash[24895]: cluster 2024-09-13T00:31:41.788045+0000 mgr.y (mgr.14397) 100 : cluster [DBG] pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:43.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:42 smithi088 bash[24895]: audit 2024-09-13T00:31:42.830197+0000 mon.c (mon.2) 71 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:43.365 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738475 got 34359738479 for osd.0 2024-09-13T00:31:43.365 DEBUG:teuthology.parallel:result is None 2024-09-13T00:31:43.525 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182473 got 107374182478 for osd.3 2024-09-13T00:31:43.525 DEBUG:teuthology.parallel:result is None 2024-09-13T00:31:43.723 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626476 got 180388626482 for osd.6 2024-09-13T00:31:43.723 DEBUG:teuthology.parallel:result is None 2024-09-13T00:31:43.971 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:43 smithi077 bash[22596]: audit 2024-09-13T00:31:43.013004+0000 mon.a (mon.0) 732 : audit [DBG] from='client.? 172.21.15.77:0/321222114' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T00:31:43.971 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:43 smithi077 bash[22596]: audit 2024-09-13T00:31:43.102656+0000 mon.c (mon.2) 72 : audit [DBG] from='client.? 172.21.15.77:0/2494891428' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T00:31:44.025 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:43 smithi077 bash[32040]: audit 2024-09-13T00:31:43.013004+0000 mon.a (mon.0) 732 : audit [DBG] from='client.? 172.21.15.77:0/321222114' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T00:31:44.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:43 smithi077 bash[32040]: audit 2024-09-13T00:31:43.102656+0000 mon.c (mon.2) 72 : audit [DBG] from='client.? 172.21.15.77:0/2494891428' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T00:31:44.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:43 smithi088 bash[24895]: audit 2024-09-13T00:31:43.013004+0000 mon.a (mon.0) 732 : audit [DBG] from='client.? 172.21.15.77:0/321222114' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T00:31:44.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:43 smithi088 bash[24895]: audit 2024-09-13T00:31:43.102656+0000 mon.c (mon.2) 72 : audit [DBG] from='client.? 172.21.15.77:0/2494891428' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T00:31:44.755 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430243 got 206158430248 for osd.7 2024-09-13T00:31:44.755 DEBUG:teuthology.parallel:result is None 2024-09-13T00:31:44.805 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574946 got 55834574951 for osd.1 2024-09-13T00:31:44.805 DEBUG:teuthology.parallel:result is None 2024-09-13T00:31:44.805 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T00:31:44.805 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T00:31:45.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:44 smithi077 bash[22596]: cluster 2024-09-13T00:31:43.788675+0000 mgr.y (mgr.14397) 101 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:45.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:44 smithi077 bash[32040]: cluster 2024-09-13T00:31:43.788675+0000 mgr.y (mgr.14397) 101 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:45.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:44 smithi088 bash[24895]: cluster 2024-09-13T00:31:43.788675+0000 mgr.y (mgr.14397) 101 : cluster [DBG] pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:47.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:46 smithi077 bash[22596]: cluster 2024-09-13T00:31:45.789368+0000 mgr.y (mgr.14397) 102 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:47.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:46 smithi077 bash[32040]: cluster 2024-09-13T00:31:45.789368+0000 mgr.y (mgr.14397) 102 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:47.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:46 smithi088 bash[24895]: cluster 2024-09-13T00:31:45.789368+0000 mgr.y (mgr.14397) 102 : cluster [DBG] pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:49.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:48 smithi077 bash[22596]: cluster 2024-09-13T00:31:47.789914+0000 mgr.y (mgr.14397) 103 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:49.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:48 smithi077 bash[32040]: cluster 2024-09-13T00:31:47.789914+0000 mgr.y (mgr.14397) 103 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:49.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:48 smithi088 bash[24895]: cluster 2024-09-13T00:31:47.789914+0000 mgr.y (mgr.14397) 103 : cluster [DBG] pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:49.580 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:31:51.099 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:50 smithi088 bash[24895]: cluster 2024-09-13T00:31:49.790681+0000 mgr.y (mgr.14397) 104 : cluster [DBG] pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:51.099 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:31:50 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:31:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:31:51.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:50 smithi077 bash[22596]: cluster 2024-09-13T00:31:49.790681+0000 mgr.y (mgr.14397) 104 : cluster [DBG] pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:51.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:50 smithi077 bash[32040]: cluster 2024-09-13T00:31:49.790681+0000 mgr.y (mgr.14397) 104 : cluster [DBG] pgmap v81: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:51.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:31:51 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:31:51] "GET /metrics HTTP/1.1" 200 203937 "" "Prometheus/2.43.0" 2024-09-13T00:31:52.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:51 smithi077 bash[22596]: audit 2024-09-13T00:31:51.342796+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:31:52.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:51 smithi077 bash[22596]: audit 2024-09-13T00:31:51.809253+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:31:52.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:51 smithi077 bash[22596]: audit 2024-09-13T00:31:51.810965+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:31:52.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:52 smithi077 bash[22596]: audit 2024-09-13T00:31:51.821360+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:31:52.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:52 smithi077 bash[32040]: audit 2024-09-13T00:31:51.342796+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:31:52.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:52 smithi077 bash[32040]: audit 2024-09-13T00:31:51.809253+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:31:52.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:52 smithi077 bash[32040]: audit 2024-09-13T00:31:51.810965+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:31:52.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:52 smithi077 bash[32040]: audit 2024-09-13T00:31:51.821360+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:31:52.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:51 smithi088 bash[24895]: audit 2024-09-13T00:31:51.342796+0000 mon.c (mon.2) 73 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:31:52.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:51 smithi088 bash[24895]: audit 2024-09-13T00:31:51.809253+0000 mon.c (mon.2) 74 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:31:52.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:51 smithi088 bash[24895]: audit 2024-09-13T00:31:51.810965+0000 mon.c (mon.2) 75 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:31:52.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:51 smithi088 bash[24895]: audit 2024-09-13T00:31:51.821360+0000 mon.a (mon.0) 733 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:31:53.238 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:52 smithi077 bash[22596]: cluster 2024-09-13T00:31:51.791278+0000 mgr.y (mgr.14397) 105 : cluster [DBG] pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:53.239 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:52 smithi077 bash[32040]: cluster 2024-09-13T00:31:51.791278+0000 mgr.y (mgr.14397) 105 : cluster [DBG] pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:53.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:52 smithi088 bash[24895]: cluster 2024-09-13T00:31:51.791278+0000 mgr.y (mgr.14397) 105 : cluster [DBG] pgmap v82: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:53.545 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:31:53.546 INFO:teuthology.orchestra.run.smithi077.stderr:dumped all 2024-09-13T00:31:54.331 INFO:teuthology.orchestra.run.smithi077.stdout:{"pg_ready":true,"pg_map":{"version":82,"stamp":"2024-09-13T00:31:51.790982+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":115,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82,"ondisk_log_size":82,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385096,"kb_used_data":6248,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576120,"statfs":{"total":767960285184,"available":765517946880,"internally_reserved":0,"allocated":6397952,"data_stored":3716195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003769"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T00:30:30.865078+0000","last_change":"2024-09-13T00:30:22.062239+0000","last_active":"2024-09-13T00:30:30.865078+0000","last_peered":"2024-09-13T00:30:30.865078+0000","last_clean":"2024-09-13T00:30:30.865078+0000","last_became_active":"2024-09-13T00:30:11.000458+0000","last_became_peered":"2024-09-13T00:30:11.000458+0000","last_unstale":"2024-09-13T00:30:30.865078+0000","last_undegraded":"2024-09-13T00:30:30.865078+0000","last_fullsized":"2024-09-13T00:30:30.865078+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:59:58.769064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00135603,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T00:30:30.842219+0000","last_change":"2024-09-13T00:30:22.061997+0000","last_active":"2024-09-13T00:30:30.842219+0000","last_peered":"2024-09-13T00:30:30.842219+0000","last_clean":"2024-09-13T00:30:30.842219+0000","last_became_active":"2024-09-13T00:30:11.064525+0000","last_became_peered":"2024-09-13T00:30:11.064525+0000","last_unstale":"2024-09-13T00:30:30.842219+0000","last_undegraded":"2024-09-13T00:30:30.842219+0000","last_fullsized":"2024-09-13T00:30:30.842219+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:13:46.391207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085995799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T00:30:30.881990+0000","last_change":"2024-09-13T00:30:22.179990+0000","last_active":"2024-09-13T00:30:30.881990+0000","last_peered":"2024-09-13T00:30:30.881990+0000","last_clean":"2024-09-13T00:30:30.881990+0000","last_became_active":"2024-09-13T00:30:11.064377+0000","last_became_peered":"2024-09-13T00:30:11.064377+0000","last_unstale":"2024-09-13T00:30:30.881990+0000","last_undegraded":"2024-09-13T00:30:30.881990+0000","last_fullsized":"2024-09-13T00:30:30.881990+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:35:54.462078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077602300000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T00:30:30.842174+0000","last_change":"2024-09-13T00:30:22.061674+0000","last_active":"2024-09-13T00:30:30.842174+0000","last_peered":"2024-09-13T00:30:30.842174+0000","last_clean":"2024-09-13T00:30:30.842174+0000","last_became_active":"2024-09-13T00:30:11.063404+0000","last_became_peered":"2024-09-13T00:30:11.063404+0000","last_unstale":"2024-09-13T00:30:30.842174+0000","last_undegraded":"2024-09-13T00:30:30.842174+0000","last_fullsized":"2024-09-13T00:30:30.842174+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:32:31.224759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053219400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"57'2","reported_seq":21,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T00:30:30.881328+0000","last_change":"2024-09-13T00:30:22.059692+0000","last_active":"2024-09-13T00:30:30.881328+0000","last_peered":"2024-09-13T00:30:30.881328+0000","last_clean":"2024-09-13T00:30:30.881328+0000","last_became_active":"2024-09-13T00:30:11.001662+0000","last_became_peered":"2024-09-13T00:30:11.001662+0000","last_unstale":"2024-09-13T00:30:30.881328+0000","last_undegraded":"2024-09-13T00:30:30.881328+0000","last_fullsized":"2024-09-13T00:30:30.881328+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:11:06.803316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002059707,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T00:30:30.880615+0000","last_change":"2024-09-13T00:30:22.048961+0000","last_active":"2024-09-13T00:30:30.880615+0000","last_peered":"2024-09-13T00:30:30.880615+0000","last_clean":"2024-09-13T00:30:30.880615+0000","last_became_active":"2024-09-13T00:30:11.063178+0000","last_became_peered":"2024-09-13T00:30:11.063178+0000","last_unstale":"2024-09-13T00:30:30.880615+0000","last_undegraded":"2024-09-13T00:30:30.880615+0000","last_fullsized":"2024-09-13T00:30:30.880615+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:15:51.153544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067555600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T00:30:30.882132+0000","last_change":"2024-09-13T00:30:22.179791+0000","last_active":"2024-09-13T00:30:30.882132+0000","last_peered":"2024-09-13T00:30:30.882132+0000","last_clean":"2024-09-13T00:30:30.882132+0000","last_became_active":"2024-09-13T00:30:11.064343+0000","last_became_peered":"2024-09-13T00:30:11.064343+0000","last_unstale":"2024-09-13T00:30:30.882132+0000","last_undegraded":"2024-09-13T00:30:30.882132+0000","last_fullsized":"2024-09-13T00:30:30.882132+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:40:42.486916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062469399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"55'1","reported_seq":20,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T00:30:30.881340+0000","last_change":"2024-09-13T00:30:22.058593+0000","last_active":"2024-09-13T00:30:30.881340+0000","last_peered":"2024-09-13T00:30:30.881340+0000","last_clean":"2024-09-13T00:30:30.881340+0000","last_became_active":"2024-09-13T00:30:11.000634+0000","last_became_peered":"2024-09-13T00:30:11.000634+0000","last_unstale":"2024-09-13T00:30:30.881340+0000","last_undegraded":"2024-09-13T00:30:30.881340+0000","last_fullsized":"2024-09-13T00:30:30.881340+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:18:10.848845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096918099999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"51'79","reported_seq":49,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T00:30:22.179174+0000","last_change":"2024-09-13T00:28:31.831547+0000","last_active":"2024-09-13T00:30:22.179174+0000","last_peered":"2024-09-13T00:30:22.179174+0000","last_clean":"2024-09-13T00:30:22.179174+0000","last_became_active":"2024-09-13T00:28:31.819822+0000","last_became_peered":"2024-09-13T00:28:31.819822+0000","last_unstale":"2024-09-13T00:30:22.179174+0000","last_undegraded":"2024-09-13T00:30:22.179174+0000","last_fullsized":"2024-09-13T00:30:22.179174+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:24:16.220413+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:24:16.220413+0000","last_clean_scrub_stamp":"2024-09-13T00:24:16.220413+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:19:22.270602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430250,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298500,"kb_used_data":1144,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446652,"statfs":{"total":95995035648,"available":95689371648,"internally_reserved":0,"allocated":1171456,"data_stored":833637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.88300000000000001,"15min":0.85699999999999998},"min":{"1min":0.63400000000000001,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.119,"5min":1.345,"15min":1.345},"last":0.92400000000000004},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.86499999999999999,"15min":0.85899999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.2709999999999999,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":1.083}]},{"osd":1,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.83199999999999996,"15min":0.82099999999999995},"min":{"1min":0.59099999999999997,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.1559999999999999,"5min":1.319,"15min":1.319},"last":0.94799999999999995},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.86699999999999999,"15min":0.84099999999999997},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.327,"5min":1.385,"15min":1.385},"last":1.181}]},{"osd":2,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.92300000000000004,"15min":0.91900000000000004},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.2769999999999999,"5min":1.45,"15min":1.45},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.90200000000000002,"15min":0.89200000000000002},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.2390000000000001,"5min":1.2390000000000001,"15min":1.2390000000000001},"last":1.1220000000000001}]},{"osd":3,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.89900000000000002,"15min":0.875},"min":{"1min":0.60799999999999998,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.248,"5min":1.365,"15min":1.365},"last":1.0600000000000001},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.94499999999999995,"15min":0.94099999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.1919999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.85799999999999998}]},{"osd":4,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.89300000000000002,"15min":0.89700000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.161,"5min":1.4790000000000001,"15min":1.4790000000000001},"last":0.88300000000000001},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.93600000000000005,"15min":0.89400000000000002},"min":{"1min":0.377,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.25,"5min":1.6160000000000001,"15min":1.6160000000000001},"last":1.151}]},{"osd":5,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.89600000000000002,"15min":0.88400000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.2230000000000001,"5min":1.506,"15min":1.506},"last":1.034},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.95799999999999996,"15min":0.93600000000000005},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.3169999999999999,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":0.71799999999999997}]},{"osd":6,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":0.97999999999999998,"15min":0.96599999999999997},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.4059999999999999,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":0.96799999999999997},{"interface":"front","average":{"1min":1.036,"5min":1.0449999999999999,"15min":1.0289999999999999},"min":{"1min":0.42799999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.2869999999999999,"5min":1.742,"15min":1.742},"last":1.002}]}]},{"osd":6,"up_from":42,"seq":180388626484,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298496,"kb_used_data":1140,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446656,"statfs":{"total":95995035648,"available":95689375744,"internally_reserved":0,"allocated":1167360,"data_stored":833097,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.84899999999999998,"15min":0.83099999999999996},"min":{"1min":0.63800000000000001,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.097,"5min":1.23,"15min":1.23},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.88800000000000001,"15min":0.86199999999999999},"min":{"1min":0.61099999999999999,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.2410000000000001,"5min":1.246,"15min":1.246},"last":0.96599999999999997}]},{"osd":1,"last update":"Fri Sep 13 00:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.90500000000000003,"15min":0.92100000000000004},"min":{"1min":0.64100000000000001,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.1539999999999999,"5min":1.3080000000000001,"15min":1.3080000000000001},"last":0.80500000000000005},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.89400000000000002,"15min":0.89000000000000001},"min":{"1min":0.59199999999999997,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.107,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":1.014}]},{"osd":2,"last update":"Fri Sep 13 00:31:48 2024","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":0.93400000000000005,"15min":0.90800000000000003},"min":{"1min":0.72999999999999998,"5min":0.443,"15min":0.443},"max":{"1min":1.333,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.196},{"interface":"front","average":{"1min":0.88,"5min":0.89700000000000002,"15min":0.91300000000000003},"min":{"1min":0.56399999999999995,"5min":0.503,"15min":0.503},"max":{"1min":1.085,"5min":1.194,"15min":1.194},"last":0.93300000000000005}]},{"osd":3,"last update":"Fri Sep 13 00:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.89700000000000002,"15min":0.91100000000000003},"min":{"1min":0.67500000000000004,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.0700000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.032,"5min":0.98399999999999999,"15min":0.95099999999999996},"min":{"1min":0.78800000000000003,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.1799999999999999,"5min":1.3680000000000001,"15min":1.3680000000000001},"last":1.0860000000000001}]},{"osd":4,"last update":"Fri Sep 13 00:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.82999999999999996,"15min":0.82599999999999996},"min":{"1min":0.59699999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.05,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":1.05},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.90800000000000003,"15min":0.877},"min":{"1min":0.64200000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.1830000000000001,"5min":1.403,"15min":1.403},"last":1.127}]},{"osd":5,"last update":"Fri Sep 13 00:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.90000000000000002,"15min":0.84599999999999997},"min":{"1min":0.66300000000000003,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.357,"5min":1.357,"15min":1.357},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.94199999999999995,"15min":0.93200000000000005},"min":{"1min":0.69599999999999995,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.177,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":1.1539999999999999}]},{"osd":7,"last update":"Fri Sep 13 00:31:37 2024","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":0.98099999999999998,"15min":0.95399999999999996},"min":{"1min":0.81799999999999995,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.367,"5min":1.391,"15min":1.391},"last":1.274},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.079,"15min":1.0549999999999999},"min":{"1min":0.79700000000000004,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.4379999999999999,"5min":1.494,"15min":1.494},"last":1.343}]}]},{"osd":5,"up_from":36,"seq":154618822718,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297916,"kb_used_data":560,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447236,"statfs":{"total":95995035648,"available":95689969664,"internally_reserved":0,"allocated":573440,"data_stored":242729,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.83799999999999997,"15min":0.81899999999999995},"min":{"1min":0.495,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":1.167},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.84299999999999997,"15min":0.82999999999999996},"min":{"1min":0.52200000000000002,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.2350000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.202}]},{"osd":1,"last update":"Fri Sep 13 00:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.87,"15min":0.83399999999999996},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.351,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":0.97399999999999998},{"interface":"front","average":{"1min":0.85999999999999999,"5min":0.86299999999999999,"15min":0.84699999999999998},"min":{"1min":0.46899999999999997,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.516,"5min":1.516,"15min":1.516},"last":1.012}]},{"osd":2,"last update":"Fri Sep 13 00:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.85599999999999998,"15min":0.84399999999999997},"min":{"1min":0.53700000000000003,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.298,"5min":1.306,"15min":1.306},"last":0.90500000000000003},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.91200000000000003,"15min":0.85099999999999998},"min":{"1min":0.624,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.323,"5min":3.556,"15min":3.556},"last":0.88100000000000001}]},{"osd":3,"last update":"Fri Sep 13 00:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.88600000000000001,"15min":0.85999999999999999},"min":{"1min":0.65700000000000003,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.3700000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.2430000000000001},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.90000000000000002,"15min":0.84899999999999998},"min":{"1min":0.58899999999999997,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.456,"5min":1.456,"15min":1.456},"last":1.087}]},{"osd":4,"last update":"Fri Sep 13 00:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.747,"15min":0.66900000000000004},"min":{"1min":0.49199999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.206,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.80900000000000005,"15min":0.70999999999999996},"min":{"1min":0.504,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.4590000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":1.05}]},{"osd":6,"last update":"Fri Sep 13 00:30:48 2024","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.86299999999999999,"15min":0.84599999999999997},"min":{"1min":0.56499999999999995,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.2529999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.141},{"interface":"front","average":{"1min":1.052,"5min":0.96399999999999997,"15min":0.93100000000000005},"min":{"1min":0.45700000000000002,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.603,"5min":1.603,"15min":1.603},"last":1.276}]},{"osd":7,"last update":"Fri Sep 13 00:31:38 2024","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.087,"15min":1.0840000000000001},"min":{"1min":0.64700000000000002,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.3680000000000001,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.323},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.0169999999999999,"15min":1.0309999999999999},"min":{"1min":0.61699999999999999,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.29,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.80700000000000005}]}]},{"osd":4,"up_from":30,"seq":128849018951,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297920,"kb_used_data":564,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447232,"statfs":{"total":95995035648,"available":95689965568,"internally_reserved":0,"allocated":577536,"data_stored":243269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.83699999999999997,"15min":0.78900000000000003},"min":{"1min":0.54100000000000004,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.167,"5min":1.216,"15min":1.216},"last":1.008},{"interface":"front","average":{"1min":0.89800000000000002,"5min":0.86599999999999999,"15min":0.81799999999999995},"min":{"1min":0.495,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.284,"5min":1.284,"15min":1.284},"last":1.2589999999999999}]},{"osd":1,"last update":"Fri Sep 13 00:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.85499999999999998,"15min":0.81499999999999995},"min":{"1min":0.55700000000000005,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.24,"5min":1.405,"15min":1.405},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.89700000000000002,"15min":0.83699999999999997},"min":{"1min":0.71399999999999997,"5min":0.438,"15min":0.438},"max":{"1min":1.177,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":1.0820000000000001}]},{"osd":2,"last update":"Fri Sep 13 00:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.878,"15min":0.83399999999999996},"min":{"1min":0.67800000000000005,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.165,"5min":1.194,"15min":1.194},"last":0.753},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.871,"15min":0.83699999999999997},"min":{"1min":0.53800000000000003,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.21,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.83199999999999996}]},{"osd":3,"last update":"Fri Sep 13 00:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.86099999999999999,"15min":0.89100000000000001},"min":{"1min":0.60699999999999998,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.143,"5min":1.587,"15min":1.587},"last":1.0489999999999999},{"interface":"front","average":{"1min":0.998,"5min":0.91600000000000004,"15min":0.88100000000000001},"min":{"1min":0.69999999999999996,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.2410000000000001,"5min":1.2929999999999999,"15min":1.2929999999999999},"last":1.1579999999999999}]},{"osd":5,"last update":"Fri Sep 13 00:31:00 2024","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":0.78500000000000003,"15min":0.67900000000000005},"min":{"1min":0.39600000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.405,"5min":1.405,"15min":1.405},"last":1.1850000000000001},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.79800000000000004,"15min":0.71099999999999997},"min":{"1min":0.35199999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.361,"5min":1.361,"15min":1.361},"last":0.96599999999999997}]},{"osd":6,"last update":"Fri Sep 13 00:31:45 2024","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.88700000000000001,"15min":0.82499999999999996},"min":{"1min":0.41099999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.347,"5min":1.347,"15min":1.347},"last":1.2410000000000001},{"interface":"front","average":{"1min":1.0289999999999999,"5min":0.94499999999999995,"15min":0.88900000000000001},"min":{"1min":0.42299999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.1930000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":1.349}]},{"osd":7,"last update":"Fri Sep 13 00:31:34 2024","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":1.0029999999999999,"15min":1.022},"min":{"1min":0.38500000000000001,"5min":0.37,"15min":0.37},"max":{"1min":1.25,"5min":1.516,"15min":1.516},"last":1.1299999999999999},{"interface":"front","average":{"1min":1.014,"5min":1.018,"15min":1.018},"min":{"1min":0.442,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.3240000000000001,"5min":1.464,"15min":1.464},"last":1.3149999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182481,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297920,"kb_used_data":564,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447232,"statfs":{"total":95995035648,"available":95689965568,"internally_reserved":0,"allocated":577536,"data_stored":243269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:31:20 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.65100000000000002,"15min":0.63800000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.89700000000000002,"5min":1.1279999999999999,"15min":1.1279999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.71499999999999997,"15min":0.74399999999999999},"min":{"1min":0.36399999999999999,"5min":0.307,"15min":0.307},"max":{"1min":1.0609999999999999,"5min":1.204,"15min":2.1400000000000001},"last":0.82899999999999996}]},{"osd":1,"last update":"Fri Sep 13 00:31:20 2024","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.754,"15min":0.76400000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.95099999999999996,"5min":1.1319999999999999,"15min":2.2029999999999998},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.72899999999999998,"15min":0.70899999999999996},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.92100000000000004,"5min":1.274,"15min":1.274},"last":0.94599999999999995}]},{"osd":2,"last update":"Fri Sep 13 00:31:20 2024","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.79500000000000004,"15min":0.76500000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.046,"5min":1.282,"15min":1.282},"last":0.86699999999999999},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.74199999999999999,"15min":0.70699999999999996},"min":{"1min":0.34699999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.006,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.80100000000000005}]},{"osd":4,"last update":"Fri Sep 13 00:31:14 2024","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.89700000000000002,"15min":0.88100000000000001},"min":{"1min":0.59499999999999997,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.127,"5min":1.153,"15min":1.153},"last":0.91400000000000003},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.95299999999999996,"15min":0.90100000000000002},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.294,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.97299999999999998}]},{"osd":5,"last update":"Fri Sep 13 00:31:00 2024","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.97699999999999998,"15min":0.95599999999999996},"min":{"1min":0.67900000000000005,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.218,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.96699999999999997,"15min":0.94699999999999995},"min":{"1min":0.73699999999999999,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.3740000000000001,"5min":1.732,"15min":1.732},"last":1.038}]},{"osd":6,"last update":"Fri Sep 13 00:31:47 2024","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.0580000000000001,"15min":1.0960000000000001},"min":{"1min":0.64900000000000002,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.194,"5min":2.6040000000000001,"15min":2.6040000000000001},"last":1.081},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.028,"15min":1.034},"min":{"1min":0.61199999999999999,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":1.2909999999999999,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":1.117}]},{"osd":7,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":1.0629999999999999,"15min":1.105},"min":{"1min":0.61099999999999999,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.246,"5min":1.619,"15min":1.619},"last":1.1619999999999999},{"interface":"front","average":{"1min":0.91000000000000003,"5min":1.0429999999999999,"15min":1.083},"min":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.22,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":1.014}]}]},{"osd":2,"up_from":18,"seq":77309411421,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297920,"kb_used_data":564,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447232,"statfs":{"total":95995035648,"available":95689965568,"internally_reserved":0,"allocated":577536,"data_stored":243269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:31:28 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.67100000000000004,"15min":0.56499999999999995},"min":{"1min":0.40200000000000002,"5min":0.36099999999999999,"15min":0.30199999999999999},"max":{"1min":0.95399999999999996,"5min":1.087,"15min":1.087},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.626,"5min":0.67500000000000004,"15min":0.57499999999999996},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.73299999999999998}]},{"osd":1,"last update":"Fri Sep 13 00:31:28 2024","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.73699999999999999,"15min":0.64100000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.33800000000000002},"max":{"1min":1.0680000000000001,"5min":1.139,"15min":1.155},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.751,"5min":0.74299999999999999,"15min":0.626},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.26500000000000001},"max":{"1min":1.8680000000000001,"5min":1.8680000000000001,"15min":1.8680000000000001},"last":0.67800000000000005}]},{"osd":3,"last update":"Fri Sep 13 00:31:18 2024","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.81599999999999995,"15min":0.75900000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.3580000000000001,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.76500000000000001,"5min":0.82399999999999995,"15min":0.80000000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.099,"5min":1.319,"15min":1.319},"last":0.90900000000000003}]},{"osd":4,"last update":"Fri Sep 13 00:31:17 2024","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.95099999999999996,"15min":0.92000000000000004},"min":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.1659999999999999,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.046},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.998,"15min":0.92600000000000005},"min":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"max":{"1min":1.2509999999999999,"5min":1.6599999999999999,"15min":1.6599999999999999},"last":1.018}]},{"osd":5,"last update":"Fri Sep 13 00:31:01 2024","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.96799999999999997,"15min":0.94999999999999996},"min":{"1min":0.70899999999999996,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.2330000000000001,"5min":1.403,"15min":1.403},"last":0.94299999999999995},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.92200000000000004,"15min":0.90200000000000002},"min":{"1min":0.63,"5min":0.63,"15min":0.63},"max":{"1min":1.224,"5min":1.304,"15min":1.304},"last":1.1419999999999999}]},{"osd":6,"last update":"Fri Sep 13 00:30:49 2024","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.98499999999999999},"min":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"max":{"1min":1.262,"5min":1.345,"15min":1.345},"last":1.077},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.0429999999999999,"15min":1.0289999999999999},"min":{"1min":0.90100000000000002,"5min":0.70399999999999996,"15min":0.70399999999999996},"max":{"1min":1.339,"5min":1.4790000000000001,"15min":1.4790000000000001},"last":0.875}]},{"osd":7,"last update":"Fri Sep 13 00:31:34 2024","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.077,"15min":1.0820000000000001},"min":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"max":{"1min":1.4319999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":1.111},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.071,"15min":1.087},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.45,"5min":1.6220000000000001,"15min":1.6220000000000001},"last":0.96599999999999997}]}]},{"osd":1,"up_from":13,"seq":55834574953,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297924,"kb_used_data":568,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447228,"statfs":{"total":95995035648,"available":95689961472,"internally_reserved":0,"allocated":581632,"data_stored":243288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:31:43 2024","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.65400000000000003,"15min":0.53800000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.25900000000000001},"max":{"1min":0.89300000000000002,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.89300000000000002},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.76500000000000001,"15min":0.60899999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.28299999999999997},"max":{"1min":0.97899999999999998,"5min":1.8879999999999999,"15min":1.8879999999999999},"last":0.505}]},{"osd":2,"last update":"Fri Sep 13 00:31:22 2024","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.751,"15min":0.61399999999999999},"min":{"1min":0.41699999999999998,"5min":0.38100000000000001,"15min":0.28699999999999998},"max":{"1min":1.0629999999999999,"5min":1.127,"15min":1.127},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.69499999999999995,"15min":0.60699999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32000000000000001},"max":{"1min":0.93200000000000005,"5min":1.123,"15min":1.123},"last":0.72299999999999998}]},{"osd":3,"last update":"Fri Sep 13 00:31:22 2024","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.73899999999999999,"15min":0.71299999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.93799999999999994,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.81599999999999995,"5min":0.85999999999999999,"15min":0.78000000000000003},"min":{"1min":0.504,"5min":0.504,"15min":0.45900000000000002},"max":{"1min":1.105,"5min":1.7809999999999999,"15min":1.7809999999999999},"last":0.92600000000000005}]},{"osd":4,"last update":"Fri Sep 13 00:31:13 2024","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.0169999999999999,"15min":0.95099999999999996},"min":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"max":{"1min":1.7809999999999999,"5min":1.7809999999999999,"15min":1.7809999999999999},"last":0.995},{"interface":"front","average":{"1min":1.008,"5min":0.93500000000000005,"15min":0.86599999999999999},"min":{"1min":0.66000000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.633,"5min":1.633,"15min":1.633},"last":1.032}]},{"osd":5,"last update":"Fri Sep 13 00:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.91400000000000003,"15min":0.91200000000000003},"min":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":1.0780000000000001},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.93999999999999995,"15min":0.91800000000000004},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.286,"5min":1.371,"15min":1.371},"last":0.78800000000000003}]},{"osd":6,"last update":"Fri Sep 13 00:30:50 2024","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":1.0469999999999999,"15min":1.0569999999999999},"min":{"1min":0.625,"5min":0.625,"15min":0.625},"max":{"1min":1.677,"5min":1.677,"15min":1.677},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.0089999999999999,"15min":1.002},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.823,"5min":1.823,"15min":1.823},"last":0.84999999999999998}]},{"osd":7,"last update":"Fri Sep 13 00:31:43 2024","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.0269999999999999,"15min":1.032},"min":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.349,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":0.96299999999999997},{"interface":"front","average":{"1min":1.077,"5min":1.0940000000000001,"15min":1.0860000000000001},"min":{"1min":0.60399999999999998,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.3380000000000001,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":1.0569999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738482,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298500,"kb_used_data":1144,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446652,"statfs":{"total":95995035648,"available":95689371648,"internally_reserved":0,"allocated":1171456,"data_stored":833637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 00:31:29 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.60599999999999998,"15min":0.52500000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.24399999999999999},"max":{"1min":0.78900000000000003,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.69299999999999995,"15min":0.56100000000000005},"min":{"1min":0.442,"5min":0.38400000000000001,"15min":0.27400000000000002},"max":{"1min":1.1799999999999999,"5min":1.2050000000000001,"15min":1.2050000000000001},"last":0.81000000000000005}]},{"osd":2,"last update":"Fri Sep 13 00:31:29 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.65900000000000003,"15min":0.59499999999999997},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.88800000000000001,"5min":1.246,"15min":1.246},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.84899999999999998,"15min":0.71399999999999997},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.33200000000000002},"max":{"1min":1.216,"5min":1.216,"15min":1.216},"last":0.93600000000000005}]},{"osd":3,"last update":"Fri Sep 13 00:31:18 2024","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.78400000000000003,"15min":0.73599999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.1819999999999999,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.90600000000000003,"15min":0.80000000000000004},"min":{"1min":0.45600000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.224,"5min":1.224,"15min":1.224},"last":1.099}]},{"osd":4,"last update":"Fri Sep 13 00:31:10 2024","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":0.98399999999999999,"15min":0.90500000000000003},"min":{"1min":0.71799999999999997,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.573,"5min":1.573,"15min":1.573},"last":1.256},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.93000000000000005,"15min":0.88500000000000001},"min":{"1min":0.61499999999999999,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.2,"5min":1.9199999999999999,"15min":1.9199999999999999},"last":0.85799999999999998}]},{"osd":5,"last update":"Fri Sep 13 00:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.878,"15min":0.85699999999999998},"min":{"1min":0.68400000000000005,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.0740000000000001,"5min":1.2490000000000001,"15min":1.2490000000000001},"last":1.01},{"interface":"front","average":{"1min":1.0469999999999999,"5min":0.97699999999999998,"15min":0.97999999999999998},"min":{"1min":0.60499999999999998,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.5,"5min":1.5,"15min":1.5},"last":1.03}]},{"osd":6,"last update":"Fri Sep 13 00:31:48 2024","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.0269999999999999,"15min":1.0409999999999999},"min":{"1min":0.68200000000000005,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.3660000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":1.2090000000000001},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.95899999999999996,"15min":0.93799999999999994},"min":{"1min":0.625,"5min":0.625,"15min":0.625},"max":{"1min":1.1639999999999999,"5min":1.99,"15min":1.99},"last":1.135}]},{"osd":7,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.119,"15min":1.131},"min":{"1min":0.78500000000000003,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.385,"5min":1.6319999999999999,"15min":1.6319999999999999},"last":1.161},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.03,"15min":1.0529999999999999},"min":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.2330000000000001,"5min":1.8280000000000001,"15min":1.8280000000000001},"last":1.0740000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T00:31:54.334 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T00:31:55.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:54 smithi077 bash[22596]: audit 2024-09-13T00:31:53.542354+0000 mgr.y (mgr.14397) 106 : audit [DBG] from='client.24452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:31:55.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:54 smithi077 bash[22596]: cluster 2024-09-13T00:31:53.791882+0000 mgr.y (mgr.14397) 107 : cluster [DBG] pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:55.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:54 smithi077 bash[32040]: audit 2024-09-13T00:31:53.542354+0000 mgr.y (mgr.14397) 106 : audit [DBG] from='client.24452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:31:55.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:54 smithi077 bash[32040]: cluster 2024-09-13T00:31:53.791882+0000 mgr.y (mgr.14397) 107 : cluster [DBG] pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:55.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:54 smithi088 bash[24895]: audit 2024-09-13T00:31:53.542354+0000 mgr.y (mgr.14397) 106 : audit [DBG] from='client.24452 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:31:55.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:54 smithi088 bash[24895]: cluster 2024-09-13T00:31:53.791882+0000 mgr.y (mgr.14397) 107 : cluster [DBG] pgmap v83: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:57.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:56 smithi077 bash[32040]: cluster 2024-09-13T00:31:55.792547+0000 mgr.y (mgr.14397) 108 : cluster [DBG] pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:57.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:56 smithi077 bash[22596]: cluster 2024-09-13T00:31:55.792547+0000 mgr.y (mgr.14397) 108 : cluster [DBG] pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:57.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:56 smithi088 bash[24895]: cluster 2024-09-13T00:31:55.792547+0000 mgr.y (mgr.14397) 108 : cluster [DBG] pgmap v84: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:58.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:58 smithi077 bash[22596]: audit 2024-09-13T00:31:57.830963+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:58.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:58 smithi077 bash[32040]: audit 2024-09-13T00:31:57.830963+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:58.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:58 smithi088 bash[24895]: audit 2024-09-13T00:31:57.830963+0000 mon.c (mon.2) 76 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:31:59.107 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:31:59.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:31:59 smithi077 bash[22596]: cluster 2024-09-13T00:31:57.793123+0000 mgr.y (mgr.14397) 109 : cluster [DBG] pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:59.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:31:59 smithi077 bash[32040]: cluster 2024-09-13T00:31:57.793123+0000 mgr.y (mgr.14397) 109 : cluster [DBG] pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:31:59.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:31:59 smithi088 bash[24895]: cluster 2024-09-13T00:31:57.793123+0000 mgr.y (mgr.14397) 109 : cluster [DBG] pgmap v85: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:01.099 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:01 smithi088 bash[24895]: cluster 2024-09-13T00:31:59.793771+0000 mgr.y (mgr.14397) 110 : cluster [DBG] pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:01.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:32:00 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:32:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:32:01.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:01 smithi077 bash[32040]: cluster 2024-09-13T00:31:59.793771+0000 mgr.y (mgr.14397) 110 : cluster [DBG] pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:01.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:01 smithi077 bash[22596]: cluster 2024-09-13T00:31:59.793771+0000 mgr.y (mgr.14397) 110 : cluster [DBG] pgmap v86: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:01.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:32:01 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:32:01] "GET /metrics HTTP/1.1" 200 203940 "" "Prometheus/2.43.0" 2024-09-13T00:32:02.950 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:32:02.951 INFO:teuthology.orchestra.run.smithi077.stderr:dumped all 2024-09-13T00:32:03.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:03 smithi077 bash[22596]: cluster 2024-09-13T00:32:01.794346+0000 mgr.y (mgr.14397) 111 : cluster [DBG] pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:03.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:03 smithi077 bash[32040]: cluster 2024-09-13T00:32:01.794346+0000 mgr.y (mgr.14397) 111 : cluster [DBG] pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:03.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:03 smithi088 bash[24895]: cluster 2024-09-13T00:32:01.794346+0000 mgr.y (mgr.14397) 111 : cluster [DBG] pgmap v87: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:03.687 INFO:teuthology.orchestra.run.smithi077.stdout:{"pg_ready":true,"pg_map":{"version":87,"stamp":"2024-09-13T00:32:01.794092+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":115,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82,"ondisk_log_size":82,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385096,"kb_used_data":6248,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747576120,"statfs":{"total":767960285184,"available":765517946880,"internally_reserved":0,"allocated":6397952,"data_stored":3716195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003870"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T00:30:30.865078+0000","last_change":"2024-09-13T00:30:22.062239+0000","last_active":"2024-09-13T00:30:30.865078+0000","last_peered":"2024-09-13T00:30:30.865078+0000","last_clean":"2024-09-13T00:30:30.865078+0000","last_became_active":"2024-09-13T00:30:11.000458+0000","last_became_peered":"2024-09-13T00:30:11.000458+0000","last_unstale":"2024-09-13T00:30:30.865078+0000","last_undegraded":"2024-09-13T00:30:30.865078+0000","last_fullsized":"2024-09-13T00:30:30.865078+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:59:58.769064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00135603,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T00:30:30.842219+0000","last_change":"2024-09-13T00:30:22.061997+0000","last_active":"2024-09-13T00:30:30.842219+0000","last_peered":"2024-09-13T00:30:30.842219+0000","last_clean":"2024-09-13T00:30:30.842219+0000","last_became_active":"2024-09-13T00:30:11.064525+0000","last_became_peered":"2024-09-13T00:30:11.064525+0000","last_unstale":"2024-09-13T00:30:30.842219+0000","last_undegraded":"2024-09-13T00:30:30.842219+0000","last_fullsized":"2024-09-13T00:30:30.842219+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:13:46.391207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085995799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T00:30:30.881990+0000","last_change":"2024-09-13T00:30:22.179990+0000","last_active":"2024-09-13T00:30:30.881990+0000","last_peered":"2024-09-13T00:30:30.881990+0000","last_clean":"2024-09-13T00:30:30.881990+0000","last_became_active":"2024-09-13T00:30:11.064377+0000","last_became_peered":"2024-09-13T00:30:11.064377+0000","last_unstale":"2024-09-13T00:30:30.881990+0000","last_undegraded":"2024-09-13T00:30:30.881990+0000","last_fullsized":"2024-09-13T00:30:30.881990+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:35:54.462078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077602300000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T00:30:30.842174+0000","last_change":"2024-09-13T00:30:22.061674+0000","last_active":"2024-09-13T00:30:30.842174+0000","last_peered":"2024-09-13T00:30:30.842174+0000","last_clean":"2024-09-13T00:30:30.842174+0000","last_became_active":"2024-09-13T00:30:11.063404+0000","last_became_peered":"2024-09-13T00:30:11.063404+0000","last_unstale":"2024-09-13T00:30:30.842174+0000","last_undegraded":"2024-09-13T00:30:30.842174+0000","last_fullsized":"2024-09-13T00:30:30.842174+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:32:31.224759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053219400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"57'2","reported_seq":21,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T00:30:30.881328+0000","last_change":"2024-09-13T00:30:22.059692+0000","last_active":"2024-09-13T00:30:30.881328+0000","last_peered":"2024-09-13T00:30:30.881328+0000","last_clean":"2024-09-13T00:30:30.881328+0000","last_became_active":"2024-09-13T00:30:11.001662+0000","last_became_peered":"2024-09-13T00:30:11.001662+0000","last_unstale":"2024-09-13T00:30:30.881328+0000","last_undegraded":"2024-09-13T00:30:30.881328+0000","last_fullsized":"2024-09-13T00:30:30.881328+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:11:06.803316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002059707,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T00:30:30.880615+0000","last_change":"2024-09-13T00:30:22.048961+0000","last_active":"2024-09-13T00:30:30.880615+0000","last_peered":"2024-09-13T00:30:30.880615+0000","last_clean":"2024-09-13T00:30:30.880615+0000","last_became_active":"2024-09-13T00:30:11.063178+0000","last_became_peered":"2024-09-13T00:30:11.063178+0000","last_unstale":"2024-09-13T00:30:30.880615+0000","last_undegraded":"2024-09-13T00:30:30.880615+0000","last_fullsized":"2024-09-13T00:30:30.880615+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:15:51.153544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067555600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T00:30:30.882132+0000","last_change":"2024-09-13T00:30:22.179791+0000","last_active":"2024-09-13T00:30:30.882132+0000","last_peered":"2024-09-13T00:30:30.882132+0000","last_clean":"2024-09-13T00:30:30.882132+0000","last_became_active":"2024-09-13T00:30:11.064343+0000","last_became_peered":"2024-09-13T00:30:11.064343+0000","last_unstale":"2024-09-13T00:30:30.882132+0000","last_undegraded":"2024-09-13T00:30:30.882132+0000","last_fullsized":"2024-09-13T00:30:30.882132+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:40:42.486916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062469399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"55'1","reported_seq":20,"reported_epoch":58,"state":"active+clean","last_fresh":"2024-09-13T00:30:30.881340+0000","last_change":"2024-09-13T00:30:22.058593+0000","last_active":"2024-09-13T00:30:30.881340+0000","last_peered":"2024-09-13T00:30:30.881340+0000","last_clean":"2024-09-13T00:30:30.881340+0000","last_became_active":"2024-09-13T00:30:11.000634+0000","last_became_peered":"2024-09-13T00:30:11.000634+0000","last_unstale":"2024-09-13T00:30:30.881340+0000","last_undegraded":"2024-09-13T00:30:30.881340+0000","last_fullsized":"2024-09-13T00:30:30.881340+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:18:10.848845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096918099999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"51'79","reported_seq":49,"reported_epoch":57,"state":"active+clean","last_fresh":"2024-09-13T00:30:22.179174+0000","last_change":"2024-09-13T00:28:31.831547+0000","last_active":"2024-09-13T00:30:22.179174+0000","last_peered":"2024-09-13T00:30:22.179174+0000","last_clean":"2024-09-13T00:30:22.179174+0000","last_became_active":"2024-09-13T00:28:31.819822+0000","last_became_peered":"2024-09-13T00:28:31.819822+0000","last_unstale":"2024-09-13T00:30:22.179174+0000","last_undegraded":"2024-09-13T00:30:22.179174+0000","last_fullsized":"2024-09-13T00:30:22.179174+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:24:16.220413+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:24:16.220413+0000","last_clean_scrub_stamp":"2024-09-13T00:24:16.220413+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:19:22.270602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430252,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298500,"kb_used_data":1144,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446652,"statfs":{"total":95995035648,"available":95689371648,"internally_reserved":0,"allocated":1171456,"data_stored":833637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.88300000000000001,"15min":0.85699999999999998},"min":{"1min":0.63400000000000001,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":1.119,"5min":1.345,"15min":1.345},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.86499999999999999,"15min":0.85899999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.2709999999999999,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":0.77700000000000002}]},{"osd":1,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":0.82799999999999996,"5min":0.83199999999999996,"15min":0.82099999999999995},"min":{"1min":0.59099999999999997,"5min":0.56899999999999995,"15min":0.56899999999999995},"max":{"1min":1.1559999999999999,"5min":1.319,"15min":1.319},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.86699999999999999,"15min":0.84099999999999997},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.327,"5min":1.385,"15min":1.385},"last":0.98399999999999999}]},{"osd":2,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.92300000000000004,"15min":0.91900000000000004},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.2769999999999999,"5min":1.45,"15min":1.45},"last":0.873},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.90200000000000002,"15min":0.89200000000000002},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.2390000000000001,"5min":1.2390000000000001,"15min":1.2390000000000001},"last":1.0389999999999999}]},{"osd":3,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":0.89900000000000002,"15min":0.875},"min":{"1min":0.60799999999999998,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.248,"5min":1.365,"15min":1.365},"last":1.01},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.94499999999999995,"15min":0.94099999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.1919999999999999,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":0.96299999999999997}]},{"osd":4,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.89300000000000002,"15min":0.89700000000000002},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.161,"5min":1.4790000000000001,"15min":1.4790000000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.93600000000000005,"15min":0.89400000000000002},"min":{"1min":0.377,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.25,"5min":1.6160000000000001,"15min":1.6160000000000001},"last":0.93500000000000005}]},{"osd":5,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.89600000000000002,"15min":0.88400000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.2230000000000001,"5min":1.506,"15min":1.506},"last":0.83199999999999996},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.95799999999999996,"15min":0.93600000000000005},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.3169999999999999,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":0.91100000000000003}]},{"osd":6,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":0.97999999999999998,"15min":0.96599999999999997},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.4059999999999999,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":0.85399999999999998},{"interface":"front","average":{"1min":1.036,"5min":1.0449999999999999,"15min":1.0289999999999999},"min":{"1min":0.42799999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.2869999999999999,"5min":1.742,"15min":1.742},"last":1.0640000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626486,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298496,"kb_used_data":1140,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446656,"statfs":{"total":95995035648,"available":95689375744,"internally_reserved":0,"allocated":1167360,"data_stored":833097,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.84899999999999998,"15min":0.83099999999999996},"min":{"1min":0.63800000000000001,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":1.097,"5min":1.23,"15min":1.23},"last":0.90500000000000003},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.88800000000000001,"15min":0.86199999999999999},"min":{"1min":0.61099999999999999,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.2410000000000001,"5min":1.246,"15min":1.246},"last":1.157}]},{"osd":1,"last update":"Fri Sep 13 00:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.88800000000000001,"5min":0.90500000000000003,"15min":0.92100000000000004},"min":{"1min":0.64100000000000001,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.1539999999999999,"5min":1.3080000000000001,"15min":1.3080000000000001},"last":1.095},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.89400000000000002,"15min":0.89000000000000001},"min":{"1min":0.59199999999999997,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.107,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.70599999999999996}]},{"osd":2,"last update":"Fri Sep 13 00:31:48 2024","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":0.93400000000000005,"15min":0.90800000000000003},"min":{"1min":0.72999999999999998,"5min":0.443,"15min":0.443},"max":{"1min":1.333,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.88,"5min":0.89700000000000002,"15min":0.91300000000000003},"min":{"1min":0.56399999999999995,"5min":0.503,"15min":0.503},"max":{"1min":1.085,"5min":1.194,"15min":1.194},"last":0.96699999999999997}]},{"osd":3,"last update":"Fri Sep 13 00:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.89700000000000002,"15min":0.91100000000000003},"min":{"1min":0.67500000000000004,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.0700000000000001,"5min":1.1850000000000001,"15min":1.1850000000000001},"last":1.022},{"interface":"front","average":{"1min":1.032,"5min":0.98399999999999999,"15min":0.95099999999999996},"min":{"1min":0.78800000000000003,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.1799999999999999,"5min":1.3680000000000001,"15min":1.3680000000000001},"last":1.351}]},{"osd":4,"last update":"Fri Sep 13 00:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.82999999999999996,"15min":0.82599999999999996},"min":{"1min":0.59699999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.05,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.90800000000000003,"15min":0.877},"min":{"1min":0.64200000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.1830000000000001,"5min":1.403,"15min":1.403},"last":1.1240000000000001}]},{"osd":5,"last update":"Fri Sep 13 00:31:48 2024","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.90000000000000002,"15min":0.84599999999999997},"min":{"1min":0.66300000000000003,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.357,"5min":1.357,"15min":1.357},"last":1.274},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.94199999999999995,"15min":0.93200000000000005},"min":{"1min":0.69599999999999995,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.177,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":1.2130000000000001}]},{"osd":7,"last update":"Fri Sep 13 00:31:37 2024","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":0.98099999999999998,"15min":0.95399999999999996},"min":{"1min":0.81799999999999995,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.367,"5min":1.391,"15min":1.391},"last":1.3959999999999999},{"interface":"front","average":{"1min":1.1060000000000001,"5min":1.079,"15min":1.0549999999999999},"min":{"1min":0.79700000000000004,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.4379999999999999,"5min":1.494,"15min":1.494},"last":1.4370000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822720,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297916,"kb_used_data":560,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447236,"statfs":{"total":95995035648,"available":95689969664,"internally_reserved":0,"allocated":573440,"data_stored":242729,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":0.83799999999999997,"15min":0.81899999999999995},"min":{"1min":0.495,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":0.88400000000000001},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.84299999999999997,"15min":0.82999999999999996},"min":{"1min":0.52200000000000002,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.2350000000000001,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.2290000000000001}]},{"osd":1,"last update":"Fri Sep 13 00:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.87,"15min":0.83399999999999996},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.351,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":1.1000000000000001},{"interface":"front","average":{"1min":0.85999999999999999,"5min":0.86299999999999999,"15min":0.84699999999999998},"min":{"1min":0.46899999999999997,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.516,"5min":1.516,"15min":1.516},"last":0.75900000000000001}]},{"osd":2,"last update":"Fri Sep 13 00:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.85599999999999998,"15min":0.84399999999999997},"min":{"1min":0.53700000000000003,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.298,"5min":1.306,"15min":1.306},"last":1.123},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.91200000000000003,"15min":0.85099999999999998},"min":{"1min":0.624,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.323,"5min":3.556,"15min":3.556},"last":0.93300000000000005}]},{"osd":3,"last update":"Fri Sep 13 00:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.88600000000000001,"15min":0.85999999999999999},"min":{"1min":0.65700000000000003,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.3700000000000001,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.0620000000000001},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.90000000000000002,"15min":0.84899999999999998},"min":{"1min":0.58899999999999997,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.456,"5min":1.456,"15min":1.456},"last":1.1870000000000001}]},{"osd":4,"last update":"Fri Sep 13 00:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.747,"15min":0.66900000000000004},"min":{"1min":0.49199999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.206,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.85899999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.80900000000000005,"15min":0.70999999999999996},"min":{"1min":0.504,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.4590000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":0.96999999999999997}]},{"osd":6,"last update":"Fri Sep 13 00:31:49 2024","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.89300000000000002,"15min":0.85599999999999998},"min":{"1min":0.63100000000000001,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.3280000000000001,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":1.024},{"interface":"front","average":{"1min":1.117,"5min":1.0049999999999999,"15min":0.94499999999999995},"min":{"1min":0.82399999999999995,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.29,"5min":1.603,"15min":1.603},"last":1.268}]},{"osd":7,"last update":"Fri Sep 13 00:31:38 2024","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.087,"15min":1.0840000000000001},"min":{"1min":0.64700000000000002,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.3680000000000001,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.1499999999999999},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.0169999999999999,"15min":1.0309999999999999},"min":{"1min":0.61699999999999999,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":1.29,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.004}]}]},{"osd":4,"up_from":30,"seq":128849018953,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297920,"kb_used_data":564,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447232,"statfs":{"total":95995035648,"available":95689965568,"internally_reserved":0,"allocated":577536,"data_stored":243269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":0.83699999999999997,"15min":0.78900000000000003},"min":{"1min":0.54100000000000004,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.167,"5min":1.216,"15min":1.216},"last":1.075},{"interface":"front","average":{"1min":0.89800000000000002,"5min":0.86599999999999999,"15min":0.81799999999999995},"min":{"1min":0.495,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":1.284,"5min":1.284,"15min":1.284},"last":0.99199999999999999}]},{"osd":1,"last update":"Fri Sep 13 00:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":0.85499999999999998,"15min":0.81499999999999995},"min":{"1min":0.55700000000000005,"5min":0.47799999999999998,"15min":0.47799999999999998},"max":{"1min":1.24,"5min":1.405,"15min":1.405},"last":0.92500000000000004},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.89700000000000002,"15min":0.83699999999999997},"min":{"1min":0.71399999999999997,"5min":0.438,"15min":0.438},"max":{"1min":1.177,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":0.89300000000000002}]},{"osd":2,"last update":"Fri Sep 13 00:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.878,"15min":0.83399999999999996},"min":{"1min":0.67800000000000005,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.165,"5min":1.194,"15min":1.194},"last":1.054},{"interface":"front","average":{"1min":0.91200000000000003,"5min":0.871,"15min":0.83699999999999997},"min":{"1min":0.53800000000000003,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.21,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":1.111}]},{"osd":3,"last update":"Fri Sep 13 00:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.86099999999999999,"15min":0.89100000000000001},"min":{"1min":0.60699999999999998,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.143,"5min":1.587,"15min":1.587},"last":1.03},{"interface":"front","average":{"1min":0.998,"5min":0.91600000000000004,"15min":0.88100000000000001},"min":{"1min":0.69999999999999996,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.2410000000000001,"5min":1.2929999999999999,"15min":1.2929999999999999},"last":1.01}]},{"osd":5,"last update":"Fri Sep 13 00:31:00 2024","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":0.78500000000000003,"15min":0.67900000000000005},"min":{"1min":0.39600000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.405,"5min":1.405,"15min":1.405},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.79800000000000004,"15min":0.71099999999999997},"min":{"1min":0.35199999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.361,"5min":1.361,"15min":1.361},"last":0.96599999999999997}]},{"osd":6,"last update":"Fri Sep 13 00:31:45 2024","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.88700000000000001,"15min":0.82499999999999996},"min":{"1min":0.41099999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.347,"5min":1.347,"15min":1.347},"last":0.71499999999999997},{"interface":"front","average":{"1min":1.0289999999999999,"5min":0.94499999999999995,"15min":0.88900000000000001},"min":{"1min":0.42299999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.1930000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":0.76900000000000002}]},{"osd":7,"last update":"Fri Sep 13 00:31:34 2024","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":1.0029999999999999,"15min":1.022},"min":{"1min":0.38500000000000001,"5min":0.37,"15min":0.37},"max":{"1min":1.25,"5min":1.516,"15min":1.516},"last":0.84999999999999998},{"interface":"front","average":{"1min":1.014,"5min":1.018,"15min":1.018},"min":{"1min":0.442,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.3240000000000001,"5min":1.464,"15min":1.464},"last":0.81799999999999995}]}]},{"osd":3,"up_from":25,"seq":107374182483,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297920,"kb_used_data":564,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447232,"statfs":{"total":95995035648,"available":95689965568,"internally_reserved":0,"allocated":577536,"data_stored":243269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:31:20 2024","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.65100000000000002,"15min":0.63800000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.89700000000000002,"5min":1.1279999999999999,"15min":1.1279999999999999},"last":1.131},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.71499999999999997,"15min":0.74399999999999999},"min":{"1min":0.36399999999999999,"5min":0.307,"15min":0.307},"max":{"1min":1.0609999999999999,"5min":1.204,"15min":2.1400000000000001},"last":1.1579999999999999}]},{"osd":1,"last update":"Fri Sep 13 00:31:20 2024","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.754,"15min":0.76400000000000001},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.95099999999999996,"5min":1.1319999999999999,"15min":2.2029999999999998},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.72899999999999998,"15min":0.70899999999999996},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.92100000000000004,"5min":1.274,"15min":1.274},"last":0.61799999999999999}]},{"osd":2,"last update":"Fri Sep 13 00:31:20 2024","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.79500000000000004,"15min":0.76500000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.046,"5min":1.282,"15min":1.282},"last":0.877},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.74199999999999999,"15min":0.70699999999999996},"min":{"1min":0.34699999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.006,"5min":1.1619999999999999,"15min":1.1619999999999999},"last":0.66200000000000003}]},{"osd":4,"last update":"Fri Sep 13 00:31:14 2024","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.89700000000000002,"15min":0.88100000000000001},"min":{"1min":0.59499999999999997,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.127,"5min":1.153,"15min":1.153},"last":0.754},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.95299999999999996,"15min":0.90100000000000002},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.294,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":0.91400000000000003}]},{"osd":5,"last update":"Fri Sep 13 00:31:00 2024","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":0.97699999999999998,"15min":0.95599999999999996},"min":{"1min":0.67900000000000005,"5min":0.63500000000000001,"15min":0.63500000000000001},"max":{"1min":1.218,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":1.1040000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.96699999999999997,"15min":0.94699999999999995},"min":{"1min":0.73699999999999999,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.3740000000000001,"5min":1.732,"15min":1.732},"last":0.77400000000000002}]},{"osd":6,"last update":"Fri Sep 13 00:31:47 2024","interfaces":[{"interface":"back","average":{"1min":0.95499999999999996,"5min":1.0580000000000001,"15min":1.0960000000000001},"min":{"1min":0.64900000000000002,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.194,"5min":2.6040000000000001,"15min":2.6040000000000001},"last":0.96699999999999997},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.028,"15min":1.034},"min":{"1min":0.61199999999999999,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":1.2909999999999999,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":0.999}]},{"osd":7,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":1.0629999999999999,"15min":1.105},"min":{"1min":0.61099999999999999,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.246,"5min":1.619,"15min":1.619},"last":1.0780000000000001},{"interface":"front","average":{"1min":0.91000000000000003,"5min":1.0429999999999999,"15min":1.083},"min":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.22,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":1.038}]}]},{"osd":2,"up_from":18,"seq":77309411423,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297920,"kb_used_data":564,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447232,"statfs":{"total":95995035648,"available":95689965568,"internally_reserved":0,"allocated":577536,"data_stored":243269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:31:28 2024","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.67100000000000004,"15min":0.56499999999999995},"min":{"1min":0.40200000000000002,"5min":0.36099999999999999,"15min":0.30199999999999999},"max":{"1min":0.95399999999999996,"5min":1.087,"15min":1.087},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.626,"5min":0.67500000000000004,"15min":0.57499999999999996},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28799999999999998},"max":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.70099999999999996}]},{"osd":1,"last update":"Fri Sep 13 00:31:28 2024","interfaces":[{"interface":"back","average":{"1min":0.748,"5min":0.73699999999999999,"15min":0.64100000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.33800000000000002},"max":{"1min":1.0680000000000001,"5min":1.139,"15min":1.155},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.751,"5min":0.74299999999999999,"15min":0.626},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.26500000000000001},"max":{"1min":1.8680000000000001,"5min":1.8680000000000001,"15min":1.8680000000000001},"last":0.81699999999999995}]},{"osd":3,"last update":"Fri Sep 13 00:31:18 2024","interfaces":[{"interface":"back","average":{"1min":0.80700000000000005,"5min":0.81599999999999995,"15min":0.75900000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.3580000000000001,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.76500000000000001,"5min":0.82399999999999995,"15min":0.80000000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.099,"5min":1.319,"15min":1.319},"last":0.95999999999999996}]},{"osd":4,"last update":"Fri Sep 13 00:31:17 2024","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.95099999999999996,"15min":0.92000000000000004},"min":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.1659999999999999,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.86099999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":0.998,"15min":0.92600000000000005},"min":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"max":{"1min":1.2509999999999999,"5min":1.6599999999999999,"15min":1.6599999999999999},"last":0.89100000000000001}]},{"osd":5,"last update":"Fri Sep 13 00:31:01 2024","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.96799999999999997,"15min":0.94999999999999996},"min":{"1min":0.70899999999999996,"5min":0.66400000000000003,"15min":0.66400000000000003},"max":{"1min":1.2330000000000001,"5min":1.403,"15min":1.403},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.92200000000000004,"15min":0.90200000000000002},"min":{"1min":0.63,"5min":0.63,"15min":0.63},"max":{"1min":1.224,"5min":1.304,"15min":1.304},"last":0.92400000000000004}]},{"osd":6,"last update":"Fri Sep 13 00:31:53 2024","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.97599999999999998,"15min":0.98499999999999999},"min":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.3069999999999999,"5min":1.345,"15min":1.345},"last":0.98999999999999999},{"interface":"front","average":{"1min":0.95799999999999996,"5min":1.03,"15min":1.0249999999999999},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.3919999999999999,"5min":1.4790000000000001,"15min":1.4790000000000001},"last":1.0089999999999999}]},{"osd":7,"last update":"Fri Sep 13 00:31:34 2024","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.077,"15min":1.0820000000000001},"min":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"max":{"1min":1.4319999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":1.0660000000000001},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.071,"15min":1.087},"min":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.45,"5min":1.6220000000000001,"15min":1.6220000000000001},"last":1.091}]}]},{"osd":1,"up_from":13,"seq":55834574955,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297924,"kb_used_data":568,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447228,"statfs":{"total":95995035648,"available":95689961472,"internally_reserved":0,"allocated":581632,"data_stored":243288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:31:43 2024","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.65400000000000003,"15min":0.53800000000000003},"min":{"1min":0.372,"5min":0.372,"15min":0.25900000000000001},"max":{"1min":0.89300000000000002,"5min":0.97999999999999998,"15min":0.97999999999999998},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.76500000000000001,"15min":0.60899999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.28299999999999997},"max":{"1min":0.97899999999999998,"5min":1.8879999999999999,"15min":1.8879999999999999},"last":0.30099999999999999}]},{"osd":2,"last update":"Fri Sep 13 00:31:22 2024","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.751,"15min":0.61399999999999999},"min":{"1min":0.41699999999999998,"5min":0.38100000000000001,"15min":0.28699999999999998},"max":{"1min":1.0629999999999999,"5min":1.127,"15min":1.127},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.69499999999999995,"15min":0.60699999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32000000000000001},"max":{"1min":0.93200000000000005,"5min":1.123,"15min":1.123},"last":0.42999999999999999}]},{"osd":3,"last update":"Fri Sep 13 00:31:22 2024","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.73899999999999999,"15min":0.71299999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.93799999999999994,"5min":1.0529999999999999,"15min":1.0529999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.81599999999999995,"5min":0.85999999999999999,"15min":0.78000000000000003},"min":{"1min":0.504,"5min":0.504,"15min":0.45900000000000002},"max":{"1min":1.105,"5min":1.7809999999999999,"15min":1.7809999999999999},"last":0.41499999999999998}]},{"osd":4,"last update":"Fri Sep 13 00:31:13 2024","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.0169999999999999,"15min":0.95099999999999996},"min":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"max":{"1min":1.7809999999999999,"5min":1.7809999999999999,"15min":1.7809999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.008,"5min":0.93500000000000005,"15min":0.86599999999999999},"min":{"1min":0.66000000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.633,"5min":1.633,"15min":1.633},"last":0.81000000000000005}]},{"osd":5,"last update":"Fri Sep 13 00:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.91400000000000003,"15min":0.91200000000000003},"min":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.96399999999999997,"5min":0.93999999999999995,"15min":0.91800000000000004},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.286,"5min":1.371,"15min":1.371},"last":0.83099999999999996}]},{"osd":6,"last update":"Fri Sep 13 00:31:53 2024","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.034,"15min":1.052},"min":{"1min":0.629,"5min":0.625,"15min":0.625},"max":{"1min":1.2909999999999999,"5min":1.677,"15min":1.677},"last":0.79600000000000004},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.996,"15min":0.998},"min":{"1min":0.66000000000000003,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.208,"5min":1.823,"15min":1.823},"last":0.66000000000000003}]},{"osd":7,"last update":"Fri Sep 13 00:31:43 2024","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.0269999999999999,"15min":1.032},"min":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.349,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":0.85599999999999998},{"interface":"front","average":{"1min":1.077,"5min":1.0940000000000001,"15min":1.0860000000000001},"min":{"1min":0.60399999999999998,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":1.3380000000000001,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":0.63200000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738484,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298500,"kb_used_data":1144,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446652,"statfs":{"total":95995035648,"available":95689371648,"internally_reserved":0,"allocated":1171456,"data_stored":833637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 00:31:29 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.60599999999999998,"15min":0.52500000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.24399999999999999},"max":{"1min":0.78900000000000003,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.878},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.69299999999999995,"15min":0.56100000000000005},"min":{"1min":0.442,"5min":0.38400000000000001,"15min":0.27400000000000002},"max":{"1min":1.1799999999999999,"5min":1.2050000000000001,"15min":1.2050000000000001},"last":0.91500000000000004}]},{"osd":2,"last update":"Fri Sep 13 00:31:29 2024","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.65900000000000003,"15min":0.59499999999999997},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.88800000000000001,"5min":1.246,"15min":1.246},"last":1.0009999999999999},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.84899999999999998,"15min":0.71399999999999997},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.33200000000000002},"max":{"1min":1.216,"5min":1.216,"15min":1.216},"last":0.66900000000000004}]},{"osd":3,"last update":"Fri Sep 13 00:31:18 2024","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.78400000000000003,"15min":0.73599999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.1819999999999999,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.1200000000000001},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.90600000000000003,"15min":0.80000000000000004},"min":{"1min":0.45600000000000002,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.224,"5min":1.224,"15min":1.224},"last":0.81399999999999995}]},{"osd":4,"last update":"Fri Sep 13 00:31:10 2024","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":0.98399999999999999,"15min":0.90500000000000003},"min":{"1min":0.71799999999999997,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.573,"5min":1.573,"15min":1.573},"last":1.087},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.93000000000000005,"15min":0.88500000000000001},"min":{"1min":0.61499999999999999,"5min":0.58999999999999997,"15min":0.58999999999999997},"max":{"1min":1.2,"5min":1.9199999999999999,"15min":1.9199999999999999},"last":1.383}]},{"osd":5,"last update":"Fri Sep 13 00:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.878,"15min":0.85699999999999998},"min":{"1min":0.68400000000000005,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.0740000000000001,"5min":1.2490000000000001,"15min":1.2490000000000001},"last":1.056},{"interface":"front","average":{"1min":1.0469999999999999,"5min":0.97699999999999998,"15min":0.97999999999999998},"min":{"1min":0.60499999999999998,"5min":0.54600000000000004,"15min":0.54600000000000004},"max":{"1min":1.5,"5min":1.5,"15min":1.5},"last":1.2150000000000001}]},{"osd":6,"last update":"Fri Sep 13 00:31:48 2024","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.0269999999999999,"15min":1.0409999999999999},"min":{"1min":0.68200000000000005,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.3660000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":1.266},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.95899999999999996,"15min":0.93799999999999994},"min":{"1min":0.625,"5min":0.625,"15min":0.625},"max":{"1min":1.1639999999999999,"5min":1.99,"15min":1.99},"last":1.2410000000000001}]},{"osd":7,"last update":"Fri Sep 13 00:31:35 2024","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":1.119,"15min":1.131},"min":{"1min":0.78500000000000003,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.385,"5min":1.6319999999999999,"15min":1.6319999999999999},"last":1.161},{"interface":"front","average":{"1min":0.95399999999999996,"5min":1.03,"15min":1.0529999999999999},"min":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.2330000000000001,"5min":1.8280000000000001,"15min":1.8280000000000001},"last":1.3129999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T00:32:03.689 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T00:32:03.690 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T00:32:03.690 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T00:32:03.690 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:32:04.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:04 smithi077 bash[22596]: audit 2024-09-13T00:32:02.952287+0000 mgr.y (mgr.14397) 112 : audit [DBG] from='client.14574 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:32:04.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:04 smithi077 bash[32040]: audit 2024-09-13T00:32:02.952287+0000 mgr.y (mgr.14397) 112 : audit [DBG] from='client.14574 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:32:04.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:04 smithi088 bash[24895]: audit 2024-09-13T00:32:02.952287+0000 mgr.y (mgr.14397) 112 : audit [DBG] from='client.14574 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:32:05.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:05 smithi077 bash[22596]: cluster 2024-09-13T00:32:03.794957+0000 mgr.y (mgr.14397) 113 : cluster [DBG] pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:05.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:05 smithi077 bash[32040]: cluster 2024-09-13T00:32:03.794957+0000 mgr.y (mgr.14397) 113 : cluster [DBG] pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:05.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:05 smithi088 bash[24895]: cluster 2024-09-13T00:32:03.794957+0000 mgr.y (mgr.14397) 113 : cluster [DBG] pgmap v88: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:07.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:07 smithi077 bash[22596]: cluster 2024-09-13T00:32:05.795545+0000 mgr.y (mgr.14397) 114 : cluster [DBG] pgmap v89: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:07.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:07 smithi077 bash[32040]: cluster 2024-09-13T00:32:05.795545+0000 mgr.y (mgr.14397) 114 : cluster [DBG] pgmap v89: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:07.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:07 smithi088 bash[24895]: cluster 2024-09-13T00:32:05.795545+0000 mgr.y (mgr.14397) 114 : cluster [DBG] pgmap v89: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:08.467 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:32:09.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:09 smithi088 bash[24895]: cluster 2024-09-13T00:32:07.796114+0000 mgr.y (mgr.14397) 115 : cluster [DBG] pgmap v90: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:09.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:09 smithi077 bash[22596]: cluster 2024-09-13T00:32:07.796114+0000 mgr.y (mgr.14397) 115 : cluster [DBG] pgmap v90: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:09.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:09 smithi077 bash[32040]: cluster 2024-09-13T00:32:07.796114+0000 mgr.y (mgr.14397) 115 : cluster [DBG] pgmap v90: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:11.034 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:32:10 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:32:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:32:11.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:11 smithi077 bash[22596]: cluster 2024-09-13T00:32:09.796706+0000 mgr.y (mgr.14397) 116 : cluster [DBG] pgmap v91: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:11.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:11 smithi077 bash[32040]: cluster 2024-09-13T00:32:09.796706+0000 mgr.y (mgr.14397) 116 : cluster [DBG] pgmap v91: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:11.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:11 smithi088 bash[24895]: cluster 2024-09-13T00:32:09.796706+0000 mgr.y (mgr.14397) 116 : cluster [DBG] pgmap v91: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:11.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:32:11 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:32:11] "GET /metrics HTTP/1.1" 200 203940 "" "Prometheus/2.43.0" 2024-09-13T00:32:12.309 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:32:12.309 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T00:32:13.057 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T00:32:13.058 INFO:tasks.cephadm:Setup complete, yielding 2024-09-13T00:32:13.058 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-13T00:32:13.069 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi077.front.sepia.ceph.com 2024-09-13T00:32:13.070 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-09-13T00:32:13.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:13 smithi088 bash[24895]: cluster 2024-09-13T00:32:11.797255+0000 mgr.y (mgr.14397) 117 : cluster [DBG] pgmap v92: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:13.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:13 smithi088 bash[24895]: audit 2024-09-13T00:32:12.314536+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.77:0/2572816753' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:32:13.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:13 smithi088 bash[24895]: audit 2024-09-13T00:32:12.831556+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:13.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:13 smithi077 bash[22596]: cluster 2024-09-13T00:32:11.797255+0000 mgr.y (mgr.14397) 117 : cluster [DBG] pgmap v92: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:13.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:13 smithi077 bash[22596]: audit 2024-09-13T00:32:12.314536+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.77:0/2572816753' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:32:13.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:13 smithi077 bash[22596]: audit 2024-09-13T00:32:12.831556+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:13.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:13 smithi077 bash[32040]: cluster 2024-09-13T00:32:11.797255+0000 mgr.y (mgr.14397) 117 : cluster [DBG] pgmap v92: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:13.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:13 smithi077 bash[32040]: audit 2024-09-13T00:32:12.314536+0000 mon.c (mon.2) 77 : audit [DBG] from='client.? 172.21.15.77:0/2572816753' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:32:13.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:13 smithi077 bash[32040]: audit 2024-09-13T00:32:12.831556+0000 mon.c (mon.2) 78 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:15.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:15 smithi088 bash[24895]: cluster 2024-09-13T00:32:13.797956+0000 mgr.y (mgr.14397) 118 : cluster [DBG] pgmap v93: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:15.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:15 smithi077 bash[22596]: cluster 2024-09-13T00:32:13.797956+0000 mgr.y (mgr.14397) 118 : cluster [DBG] pgmap v93: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:15.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:15 smithi077 bash[32040]: cluster 2024-09-13T00:32:13.797956+0000 mgr.y (mgr.14397) 118 : cluster [DBG] pgmap v93: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:17.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:17 smithi088 bash[24895]: cluster 2024-09-13T00:32:15.798526+0000 mgr.y (mgr.14397) 119 : cluster [DBG] pgmap v94: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:17.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:17 smithi077 bash[22596]: cluster 2024-09-13T00:32:15.798526+0000 mgr.y (mgr.14397) 119 : cluster [DBG] pgmap v94: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:17.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:17 smithi077 bash[32040]: cluster 2024-09-13T00:32:15.798526+0000 mgr.y (mgr.14397) 119 : cluster [DBG] pgmap v94: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:17.850 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:32:19.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:19 smithi088 bash[24895]: cluster 2024-09-13T00:32:17.799074+0000 mgr.y (mgr.14397) 120 : cluster [DBG] pgmap v95: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:19.422 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:19 smithi077 bash[22596]: cluster 2024-09-13T00:32:17.799074+0000 mgr.y (mgr.14397) 120 : cluster [DBG] pgmap v95: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:19.422 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:19 smithi077 bash[32040]: cluster 2024-09-13T00:32:17.799074+0000 mgr.y (mgr.14397) 120 : cluster [DBG] pgmap v95: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:21.057 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:32:20 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:32:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:32:21.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:21 smithi088 bash[24895]: cluster 2024-09-13T00:32:19.799664+0000 mgr.y (mgr.14397) 121 : cluster [DBG] pgmap v96: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:21.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:21 smithi077 bash[22596]: cluster 2024-09-13T00:32:19.799664+0000 mgr.y (mgr.14397) 121 : cluster [DBG] pgmap v96: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:21.461 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:21 smithi077 bash[32040]: cluster 2024-09-13T00:32:19.799664+0000 mgr.y (mgr.14397) 121 : cluster [DBG] pgmap v96: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:21.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:32:21 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:32:21] "GET /metrics HTTP/1.1" 200 203941 "" "Prometheus/2.43.0" 2024-09-13T00:32:22.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:22 smithi088 bash[24895]: audit 2024-09-13T00:32:21.477372+0000 mgr.y (mgr.14397) 122 : audit [DBG] from='client.14586 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:32:22.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:22 smithi088 bash[24895]: audit 2024-09-13T00:32:21.478090+0000 mon.c (mon.2) 79 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T00:32:22.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:22 smithi088 bash[24895]: audit 2024-09-13T00:32:21.478600+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T00:32:22.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:22 smithi077 bash[32040]: audit 2024-09-13T00:32:21.477372+0000 mgr.y (mgr.14397) 122 : audit [DBG] from='client.14586 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:32:22.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:22 smithi077 bash[32040]: audit 2024-09-13T00:32:21.478090+0000 mon.c (mon.2) 79 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T00:32:22.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:22 smithi077 bash[32040]: audit 2024-09-13T00:32:21.478600+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T00:32:22.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:22 smithi077 bash[22596]: audit 2024-09-13T00:32:21.477372+0000 mgr.y (mgr.14397) 122 : audit [DBG] from='client.14586 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:32:22.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:22 smithi077 bash[22596]: audit 2024-09-13T00:32:21.478090+0000 mon.c (mon.2) 79 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T00:32:22.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:22 smithi077 bash[22596]: audit 2024-09-13T00:32:21.478600+0000 mon.a (mon.0) 734 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-13T00:32:23.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:23 smithi088 bash[24895]: cluster 2024-09-13T00:32:21.800154+0000 mgr.y (mgr.14397) 123 : cluster [DBG] pgmap v97: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:23.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:23 smithi088 bash[24895]: audit 2024-09-13T00:32:22.081356+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-13T00:32:23.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:23 smithi088 bash[24895]: cluster 2024-09-13T00:32:22.081471+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-09-13T00:32:23.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:23 smithi088 bash[24895]: audit 2024-09-13T00:32:22.086813+0000 mon.c (mon.2) 80 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T00:32:23.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:23 smithi088 bash[24895]: audit 2024-09-13T00:32:22.090993+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T00:32:23.389 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:23 smithi077 bash[22596]: cluster 2024-09-13T00:32:21.800154+0000 mgr.y (mgr.14397) 123 : cluster [DBG] pgmap v97: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:23.389 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:23 smithi077 bash[22596]: audit 2024-09-13T00:32:22.081356+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-13T00:32:23.389 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:23 smithi077 bash[22596]: cluster 2024-09-13T00:32:22.081471+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-09-13T00:32:23.389 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:23 smithi077 bash[22596]: audit 2024-09-13T00:32:22.086813+0000 mon.c (mon.2) 80 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T00:32:23.389 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:23 smithi077 bash[22596]: audit 2024-09-13T00:32:22.090993+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T00:32:23.389 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:23 smithi077 bash[22596]: debug 2024-09-13T00:32:23.127+0000 7f6517427640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T00:32:23.389 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:23 smithi077 bash[32040]: cluster 2024-09-13T00:32:21.800154+0000 mgr.y (mgr.14397) 123 : cluster [DBG] pgmap v97: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:23.389 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:23 smithi077 bash[32040]: audit 2024-09-13T00:32:22.081356+0000 mon.a (mon.0) 735 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-13T00:32:23.389 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:23 smithi077 bash[32040]: cluster 2024-09-13T00:32:22.081471+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e59: 8 total, 8 up, 8 in 2024-09-13T00:32:23.390 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:23 smithi077 bash[32040]: audit 2024-09-13T00:32:22.086813+0000 mon.c (mon.2) 80 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T00:32:23.390 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:23 smithi077 bash[32040]: audit 2024-09-13T00:32:22.090993+0000 mon.a (mon.0) 737 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-13T00:32:24.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: audit 2024-09-13T00:32:23.090565+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-13T00:32:24.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: cluster 2024-09-13T00:32:23.091124+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-09-13T00:32:24.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: audit 2024-09-13T00:32:23.126647+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T00:32:24.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: audit 2024-09-13T00:32:23.129340+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T00:32:24.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: cluster 2024-09-13T00:32:23.134404+0000 mon.a (mon.0) 741 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T00:32:24.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: cluster 2024-09-13T00:32:23.135280+0000 mon.a (mon.0) 742 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T00:32:24.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: cluster 2024-09-13T00:32:23.143599+0000 mon.a (mon.0) 743 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-09-13T00:32:24.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: audit 2024-09-13T00:32:23.143913+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-13T00:32:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: cluster 2024-09-13T00:32:23.144033+0000 mon.a (mon.0) 745 : cluster [DBG] fsmap foo:0 2024-09-13T00:32:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: cephadm 2024-09-13T00:32:23.156159+0000 mgr.y (mgr.14397) 124 : cephadm [INF] Saving service mds.foo spec with placement count:2 2024-09-13T00:32:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: audit 2024-09-13T00:32:23.171407+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: audit 2024-09-13T00:32:23.175822+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: audit 2024-09-13T00:32:23.177980+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: audit 2024-09-13T00:32:23.179021+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:32:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: audit 2024-09-13T00:32:23.184517+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: audit 2024-09-13T00:32:23.190780+0000 mon.c (mon.2) 85 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi077.bigbbf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:32:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: audit 2024-09-13T00:32:23.191299+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi077.bigbbf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:32:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: audit 2024-09-13T00:32:23.197523+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi077.bigbbf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T00:32:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: audit 2024-09-13T00:32:23.200931+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:24 smithi088 bash[24895]: cephadm 2024-09-13T00:32:23.201807+0000 mgr.y (mgr.14397) 125 : cephadm [INF] Deploying daemon mds.foo.smithi077.bigbbf on smithi077 2024-09-13T00:32:24.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: audit 2024-09-13T00:32:23.090565+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-13T00:32:24.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: cluster 2024-09-13T00:32:23.091124+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-09-13T00:32:24.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: audit 2024-09-13T00:32:23.126647+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T00:32:24.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: audit 2024-09-13T00:32:23.129340+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T00:32:24.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: cluster 2024-09-13T00:32:23.134404+0000 mon.a (mon.0) 741 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T00:32:24.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: cluster 2024-09-13T00:32:23.135280+0000 mon.a (mon.0) 742 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T00:32:24.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: cluster 2024-09-13T00:32:23.143599+0000 mon.a (mon.0) 743 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-09-13T00:32:24.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: audit 2024-09-13T00:32:23.143913+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-13T00:32:24.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: cluster 2024-09-13T00:32:23.144033+0000 mon.a (mon.0) 745 : cluster [DBG] fsmap foo:0 2024-09-13T00:32:24.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: cephadm 2024-09-13T00:32:23.156159+0000 mgr.y (mgr.14397) 124 : cephadm [INF] Saving service mds.foo spec with placement count:2 2024-09-13T00:32:24.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: audit 2024-09-13T00:32:23.171407+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:24.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: audit 2024-09-13T00:32:23.175822+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:24.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: audit 2024-09-13T00:32:23.177980+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:24.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: audit 2024-09-13T00:32:23.179021+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:32:24.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: audit 2024-09-13T00:32:23.184517+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:24.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: audit 2024-09-13T00:32:23.190780+0000 mon.c (mon.2) 85 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi077.bigbbf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:32:24.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: audit 2024-09-13T00:32:23.191299+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi077.bigbbf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:32:24.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: audit 2024-09-13T00:32:23.197523+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi077.bigbbf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T00:32:24.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: audit 2024-09-13T00:32:23.200931+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:24.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[22596]: cephadm 2024-09-13T00:32:23.201807+0000 mgr.y (mgr.14397) 125 : cephadm [INF] Deploying daemon mds.foo.smithi077.bigbbf on smithi077 2024-09-13T00:32:24.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: audit 2024-09-13T00:32:23.090565+0000 mon.a (mon.0) 738 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-13T00:32:24.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: cluster 2024-09-13T00:32:23.091124+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e60: 8 total, 8 up, 8 in 2024-09-13T00:32:24.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: audit 2024-09-13T00:32:23.126647+0000 mon.c (mon.2) 81 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T00:32:24.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: audit 2024-09-13T00:32:23.129340+0000 mon.a (mon.0) 740 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-13T00:32:24.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: cluster 2024-09-13T00:32:23.134404+0000 mon.a (mon.0) 741 : cluster [ERR] Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-13T00:32:24.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: cluster 2024-09-13T00:32:23.135280+0000 mon.a (mon.0) 742 : cluster [WRN] Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-13T00:32:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: cluster 2024-09-13T00:32:23.143599+0000 mon.a (mon.0) 743 : cluster [DBG] osdmap e61: 8 total, 8 up, 8 in 2024-09-13T00:32:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: audit 2024-09-13T00:32:23.143913+0000 mon.a (mon.0) 744 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-13T00:32:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: cluster 2024-09-13T00:32:23.144033+0000 mon.a (mon.0) 745 : cluster [DBG] fsmap foo:0 2024-09-13T00:32:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: cephadm 2024-09-13T00:32:23.156159+0000 mgr.y (mgr.14397) 124 : cephadm [INF] Saving service mds.foo spec with placement count:2 2024-09-13T00:32:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: audit 2024-09-13T00:32:23.171407+0000 mon.a (mon.0) 746 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: audit 2024-09-13T00:32:23.175822+0000 mon.c (mon.2) 82 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: audit 2024-09-13T00:32:23.177980+0000 mon.c (mon.2) 83 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: audit 2024-09-13T00:32:23.179021+0000 mon.c (mon.2) 84 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:32:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: audit 2024-09-13T00:32:23.184517+0000 mon.a (mon.0) 747 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: audit 2024-09-13T00:32:23.190780+0000 mon.c (mon.2) 85 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi077.bigbbf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:32:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: audit 2024-09-13T00:32:23.191299+0000 mon.a (mon.0) 748 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi077.bigbbf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:32:24.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: audit 2024-09-13T00:32:23.197523+0000 mon.a (mon.0) 749 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi077.bigbbf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T00:32:24.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: audit 2024-09-13T00:32:23.200931+0000 mon.c (mon.2) 86 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:24.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:24 smithi077 bash[32040]: cephadm 2024-09-13T00:32:23.201807+0000 mgr.y (mgr.14397) 125 : cephadm [INF] Deploying daemon mds.foo.smithi077.bigbbf on smithi077 2024-09-13T00:32:24.609 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-13T00:32:25.387 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:25 smithi077 bash[22596]: cluster 2024-09-13T00:32:23.800975+0000 mgr.y (mgr.14397) 126 : cluster [DBG] pgmap v101: 73 pgs: 11 creating+activating, 62 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:25.387 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:25 smithi077 bash[22596]: cluster 2024-09-13T00:32:24.146314+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-09-13T00:32:25.388 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:25 smithi077 bash[32040]: cluster 2024-09-13T00:32:23.800975+0000 mgr.y (mgr.14397) 126 : cluster [DBG] pgmap v101: 73 pgs: 11 creating+activating, 62 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:25.388 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:25 smithi077 bash[32040]: cluster 2024-09-13T00:32:24.146314+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-09-13T00:32:25.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:25 smithi088 bash[24895]: cluster 2024-09-13T00:32:23.800975+0000 mgr.y (mgr.14397) 126 : cluster [DBG] pgmap v101: 73 pgs: 11 creating+activating, 62 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:25.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:25 smithi088 bash[24895]: cluster 2024-09-13T00:32:24.146314+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e62: 8 total, 8 up, 8 in 2024-09-13T00:32:26.624 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.624 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.624 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.625 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.625 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.625 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.626 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.626 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.626 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.887 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.888 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.888 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.888 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.889 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.889 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.889 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.890 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:26.890 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:27.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:27 smithi077 bash[32040]: cluster 2024-09-13T00:32:25.801594+0000 mgr.y (mgr.14397) 127 : cluster [DBG] pgmap v103: 73 pgs: 11 creating+activating, 62 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:27.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:27 smithi077 bash[32040]: audit 2024-09-13T00:32:26.925413+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:27.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:27 smithi077 bash[32040]: audit 2024-09-13T00:32:26.931820+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:27.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:27 smithi077 bash[32040]: audit 2024-09-13T00:32:26.937324+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:27.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:27 smithi077 bash[32040]: audit 2024-09-13T00:32:26.938645+0000 mon.c (mon.2) 87 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi088.krmowh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:32:27.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:27 smithi077 bash[32040]: audit 2024-09-13T00:32:26.938926+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi088.krmowh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:32:27.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:27 smithi077 bash[32040]: audit 2024-09-13T00:32:26.942222+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi088.krmowh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T00:32:27.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:27 smithi077 bash[32040]: audit 2024-09-13T00:32:26.943516+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:27.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:27 smithi077 bash[22596]: cluster 2024-09-13T00:32:25.801594+0000 mgr.y (mgr.14397) 127 : cluster [DBG] pgmap v103: 73 pgs: 11 creating+activating, 62 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:27.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:27 smithi077 bash[22596]: audit 2024-09-13T00:32:26.925413+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:27.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:27 smithi077 bash[22596]: audit 2024-09-13T00:32:26.931820+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:27.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:27 smithi077 bash[22596]: audit 2024-09-13T00:32:26.937324+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:27.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:27 smithi077 bash[22596]: audit 2024-09-13T00:32:26.938645+0000 mon.c (mon.2) 87 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi088.krmowh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:32:27.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:27 smithi077 bash[22596]: audit 2024-09-13T00:32:26.938926+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi088.krmowh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:32:27.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:27 smithi077 bash[22596]: audit 2024-09-13T00:32:26.942222+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi088.krmowh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T00:32:27.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:27 smithi077 bash[22596]: audit 2024-09-13T00:32:26.943516+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:27.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:27 smithi088 bash[24895]: cluster 2024-09-13T00:32:25.801594+0000 mgr.y (mgr.14397) 127 : cluster [DBG] pgmap v103: 73 pgs: 11 creating+activating, 62 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:27.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:27 smithi088 bash[24895]: audit 2024-09-13T00:32:26.925413+0000 mon.a (mon.0) 751 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:27.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:27 smithi088 bash[24895]: audit 2024-09-13T00:32:26.931820+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:27.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:27 smithi088 bash[24895]: audit 2024-09-13T00:32:26.937324+0000 mon.a (mon.0) 753 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:27.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:27 smithi088 bash[24895]: audit 2024-09-13T00:32:26.938645+0000 mon.c (mon.2) 87 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi088.krmowh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:32:27.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:27 smithi088 bash[24895]: audit 2024-09-13T00:32:26.938926+0000 mon.a (mon.0) 754 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi088.krmowh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:32:27.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:27 smithi088 bash[24895]: audit 2024-09-13T00:32:26.942222+0000 mon.a (mon.0) 755 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi088.krmowh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-13T00:32:27.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:27 smithi088 bash[24895]: audit 2024-09-13T00:32:26.943516+0000 mon.c (mon.2) 88 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:28.447 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:28 smithi077 bash[22596]: cephadm 2024-09-13T00:32:26.944235+0000 mgr.y (mgr.14397) 128 : cephadm [INF] Deploying daemon mds.foo.smithi088.krmowh on smithi088 2024-09-13T00:32:28.447 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:28 smithi077 bash[22596]: audit 2024-09-13T00:32:27.838512+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:28.448 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:28 smithi077 bash[22596]: audit 2024-09-13T00:32:27.840915+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:28.448 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:28 smithi077 bash[32040]: cephadm 2024-09-13T00:32:26.944235+0000 mgr.y (mgr.14397) 128 : cephadm [INF] Deploying daemon mds.foo.smithi088.krmowh on smithi088 2024-09-13T00:32:28.448 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:28 smithi077 bash[32040]: audit 2024-09-13T00:32:27.838512+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:28.448 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:28 smithi077 bash[32040]: audit 2024-09-13T00:32:27.840915+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:28.467 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:28 smithi088 bash[24895]: cephadm 2024-09-13T00:32:26.944235+0000 mgr.y (mgr.14397) 128 : cephadm [INF] Deploying daemon mds.foo.smithi088.krmowh on smithi088 2024-09-13T00:32:28.467 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:28 smithi088 bash[24895]: audit 2024-09-13T00:32:27.838512+0000 mon.a (mon.0) 756 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:28.467 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:28 smithi088 bash[24895]: audit 2024-09-13T00:32:27.840915+0000 mon.c (mon.2) 89 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:29.094 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:28 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.095 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:32:28 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.095 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:32:28 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.095 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:32:28 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.096 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:32:28 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.096 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:32:28 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.096 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 00:32:28 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.097 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:32:28 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.097 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:32:28 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.349 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:32:29 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.349 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 00:32:29 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.349 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:32:29 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.350 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:32:29 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:29 smithi088 bash[24895]: cluster 2024-09-13T00:32:27.802263+0000 mgr.y (mgr.14397) 129 : cluster [DBG] pgmap v104: 73 pgs: 11 creating+activating, 62 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:29.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:29 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.350 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:32:29 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.350 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:32:29 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.351 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:32:29 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.351 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:32:29 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:32:29.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:29 smithi077 bash[22596]: cluster 2024-09-13T00:32:27.802263+0000 mgr.y (mgr.14397) 129 : cluster [DBG] pgmap v104: 73 pgs: 11 creating+activating, 62 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:29.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:29 smithi077 bash[32040]: cluster 2024-09-13T00:32:27.802263+0000 mgr.y (mgr.14397) 129 : cluster [DBG] pgmap v104: 73 pgs: 11 creating+activating, 62 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:30.411 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:32:30.474 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:30 smithi088 bash[24895]: cluster 2024-09-13T00:32:29.167722+0000 mon.a (mon.0) 757 : cluster [DBG] mds.? [v2:172.21.15.77:6834/506597872,v1:172.21.15.77:6835/506597872] up:boot 2024-09-13T00:32:30.474 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:30 smithi088 bash[24895]: cluster 2024-09-13T00:32:29.167861+0000 mon.a (mon.0) 758 : cluster [INF] daemon mds.foo.smithi077.bigbbf assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-13T00:32:30.474 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:30 smithi088 bash[24895]: cluster 2024-09-13T00:32:29.168011+0000 mon.a (mon.0) 759 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T00:32:30.474 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:30 smithi088 bash[24895]: cluster 2024-09-13T00:32:29.168056+0000 mon.a (mon.0) 760 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T00:32:30.474 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:30 smithi088 bash[24895]: cluster 2024-09-13T00:32:29.168081+0000 mon.a (mon.0) 761 : cluster [INF] Cluster is now healthy 2024-09-13T00:32:30.474 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:30 smithi088 bash[24895]: cluster 2024-09-13T00:32:29.168507+0000 mon.a (mon.0) 762 : cluster [DBG] fsmap foo:0 1 up:standby 2024-09-13T00:32:30.474 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:30 smithi088 bash[24895]: audit 2024-09-13T00:32:29.168801+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi077.bigbbf"}]: dispatch 2024-09-13T00:32:30.474 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:30 smithi088 bash[24895]: cluster 2024-09-13T00:32:29.176688+0000 mon.a (mon.0) 763 : cluster [DBG] fsmap foo:1 {0=foo.smithi077.bigbbf=up:creating} 2024-09-13T00:32:30.475 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:30 smithi088 bash[24895]: cluster 2024-09-13T00:32:29.199177+0000 mon.a (mon.0) 764 : cluster [INF] daemon mds.foo.smithi077.bigbbf is now active in filesystem foo as rank 0 2024-09-13T00:32:30.475 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:30 smithi088 bash[24895]: audit 2024-09-13T00:32:29.395505+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:30.475 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:30 smithi088 bash[24895]: audit 2024-09-13T00:32:29.408702+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:30.475 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:30 smithi088 bash[24895]: audit 2024-09-13T00:32:29.419147+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:30.475 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:30 smithi088 bash[24895]: audit 2024-09-13T00:32:29.429656+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:30.475 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:30 smithi088 bash[24895]: audit 2024-09-13T00:32:29.438243+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:30.475 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:30 smithi088 bash[24895]: audit 2024-09-13T00:32:29.494217+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:30.512 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[22596]: cluster 2024-09-13T00:32:29.167722+0000 mon.a (mon.0) 757 : cluster [DBG] mds.? [v2:172.21.15.77:6834/506597872,v1:172.21.15.77:6835/506597872] up:boot 2024-09-13T00:32:30.512 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[22596]: cluster 2024-09-13T00:32:29.167861+0000 mon.a (mon.0) 758 : cluster [INF] daemon mds.foo.smithi077.bigbbf assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-13T00:32:30.512 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[22596]: cluster 2024-09-13T00:32:29.168011+0000 mon.a (mon.0) 759 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T00:32:30.512 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[22596]: cluster 2024-09-13T00:32:29.168056+0000 mon.a (mon.0) 760 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T00:32:30.512 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[22596]: cluster 2024-09-13T00:32:29.168081+0000 mon.a (mon.0) 761 : cluster [INF] Cluster is now healthy 2024-09-13T00:32:30.513 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[22596]: cluster 2024-09-13T00:32:29.168507+0000 mon.a (mon.0) 762 : cluster [DBG] fsmap foo:0 1 up:standby 2024-09-13T00:32:30.513 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[22596]: audit 2024-09-13T00:32:29.168801+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi077.bigbbf"}]: dispatch 2024-09-13T00:32:30.513 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[22596]: cluster 2024-09-13T00:32:29.176688+0000 mon.a (mon.0) 763 : cluster [DBG] fsmap foo:1 {0=foo.smithi077.bigbbf=up:creating} 2024-09-13T00:32:30.513 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[22596]: cluster 2024-09-13T00:32:29.199177+0000 mon.a (mon.0) 764 : cluster [INF] daemon mds.foo.smithi077.bigbbf is now active in filesystem foo as rank 0 2024-09-13T00:32:30.513 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[22596]: audit 2024-09-13T00:32:29.395505+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:30.513 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[22596]: audit 2024-09-13T00:32:29.408702+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:30.513 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[22596]: audit 2024-09-13T00:32:29.419147+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:30.513 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[22596]: audit 2024-09-13T00:32:29.429656+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:30.513 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[22596]: audit 2024-09-13T00:32:29.438243+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:30.513 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[22596]: audit 2024-09-13T00:32:29.494217+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:30.513 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[32040]: cluster 2024-09-13T00:32:29.167722+0000 mon.a (mon.0) 757 : cluster [DBG] mds.? [v2:172.21.15.77:6834/506597872,v1:172.21.15.77:6835/506597872] up:boot 2024-09-13T00:32:30.513 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[32040]: cluster 2024-09-13T00:32:29.167861+0000 mon.a (mon.0) 758 : cluster [INF] daemon mds.foo.smithi077.bigbbf assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-13T00:32:30.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[32040]: cluster 2024-09-13T00:32:29.168011+0000 mon.a (mon.0) 759 : cluster [INF] Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-13T00:32:30.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[32040]: cluster 2024-09-13T00:32:29.168056+0000 mon.a (mon.0) 760 : cluster [INF] Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-13T00:32:30.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[32040]: cluster 2024-09-13T00:32:29.168081+0000 mon.a (mon.0) 761 : cluster [INF] Cluster is now healthy 2024-09-13T00:32:30.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[32040]: cluster 2024-09-13T00:32:29.168507+0000 mon.a (mon.0) 762 : cluster [DBG] fsmap foo:0 1 up:standby 2024-09-13T00:32:30.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[32040]: audit 2024-09-13T00:32:29.168801+0000 mon.c (mon.2) 90 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi077.bigbbf"}]: dispatch 2024-09-13T00:32:30.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[32040]: cluster 2024-09-13T00:32:29.176688+0000 mon.a (mon.0) 763 : cluster [DBG] fsmap foo:1 {0=foo.smithi077.bigbbf=up:creating} 2024-09-13T00:32:30.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[32040]: cluster 2024-09-13T00:32:29.199177+0000 mon.a (mon.0) 764 : cluster [INF] daemon mds.foo.smithi077.bigbbf is now active in filesystem foo as rank 0 2024-09-13T00:32:30.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[32040]: audit 2024-09-13T00:32:29.395505+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:30.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[32040]: audit 2024-09-13T00:32:29.408702+0000 mon.a (mon.0) 766 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:30.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[32040]: audit 2024-09-13T00:32:29.419147+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:30.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[32040]: audit 2024-09-13T00:32:29.429656+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:30.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[32040]: audit 2024-09-13T00:32:29.438243+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:30.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:30 smithi077 bash[32040]: audit 2024-09-13T00:32:29.494217+0000 mon.c (mon.2) 91 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:31.099 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:32:30 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:32:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:32:31.459 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[32040]: cluster 2024-09-13T00:32:29.803408+0000 mgr.y (mgr.14397) 130 : cluster [DBG] pgmap v105: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:31.460 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[32040]: cluster 2024-09-13T00:32:30.180609+0000 mon.a (mon.0) 770 : cluster [DBG] mds.? [v2:172.21.15.77:6834/506597872,v1:172.21.15.77:6835/506597872] up:active 2024-09-13T00:32:31.460 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[32040]: cluster 2024-09-13T00:32:30.180724+0000 mon.a (mon.0) 771 : cluster [DBG] fsmap foo:1 {0=foo.smithi077.bigbbf=up:active} 2024-09-13T00:32:31.460 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[32040]: cluster 2024-09-13T00:32:30.861576+0000 mon.a (mon.0) 772 : cluster [INF] Dropping low affinity active daemon mds.foo.smithi077.bigbbf in favor of higher affinity standby. 2024-09-13T00:32:31.460 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[32040]: cluster 2024-09-13T00:32:30.861642+0000 mon.a (mon.0) 773 : cluster [WRN] Replacing daemon mds.foo.smithi077.bigbbf as rank 0 with standby daemon mds.foo.smithi088.krmowh 2024-09-13T00:32:31.460 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[32040]: cluster 2024-09-13T00:32:30.862219+0000 mon.a (mon.0) 774 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T00:32:31.460 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[32040]: cluster 2024-09-13T00:32:30.862258+0000 mon.a (mon.0) 775 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T00:32:31.460 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[32040]: audit 2024-09-13T00:32:30.873990+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi088.krmowh"}]: dispatch 2024-09-13T00:32:31.460 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[32040]: cluster 2024-09-13T00:32:30.874538+0000 mon.a (mon.0) 776 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-09-13T00:32:31.460 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[32040]: cluster 2024-09-13T00:32:30.874971+0000 mon.a (mon.0) 777 : cluster [DBG] mds.? [v2:172.21.15.88:6832/1144032959,v1:172.21.15.88:6833/1144032959] up:boot 2024-09-13T00:32:31.460 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[32040]: cluster 2024-09-13T00:32:30.875114+0000 mon.a (mon.0) 778 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi088.krmowh=up:replay} 2024-09-13T00:32:31.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[22596]: cluster 2024-09-13T00:32:29.803408+0000 mgr.y (mgr.14397) 130 : cluster [DBG] pgmap v105: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:31.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[22596]: cluster 2024-09-13T00:32:30.180609+0000 mon.a (mon.0) 770 : cluster [DBG] mds.? [v2:172.21.15.77:6834/506597872,v1:172.21.15.77:6835/506597872] up:active 2024-09-13T00:32:31.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[22596]: cluster 2024-09-13T00:32:30.180724+0000 mon.a (mon.0) 771 : cluster [DBG] fsmap foo:1 {0=foo.smithi077.bigbbf=up:active} 2024-09-13T00:32:31.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[22596]: cluster 2024-09-13T00:32:30.861576+0000 mon.a (mon.0) 772 : cluster [INF] Dropping low affinity active daemon mds.foo.smithi077.bigbbf in favor of higher affinity standby. 2024-09-13T00:32:31.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[22596]: cluster 2024-09-13T00:32:30.861642+0000 mon.a (mon.0) 773 : cluster [WRN] Replacing daemon mds.foo.smithi077.bigbbf as rank 0 with standby daemon mds.foo.smithi088.krmowh 2024-09-13T00:32:31.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[22596]: cluster 2024-09-13T00:32:30.862219+0000 mon.a (mon.0) 774 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T00:32:31.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[22596]: cluster 2024-09-13T00:32:30.862258+0000 mon.a (mon.0) 775 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T00:32:31.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[22596]: audit 2024-09-13T00:32:30.873990+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi088.krmowh"}]: dispatch 2024-09-13T00:32:31.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[22596]: cluster 2024-09-13T00:32:30.874538+0000 mon.a (mon.0) 776 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-09-13T00:32:31.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[22596]: cluster 2024-09-13T00:32:30.874971+0000 mon.a (mon.0) 777 : cluster [DBG] mds.? [v2:172.21.15.88:6832/1144032959,v1:172.21.15.88:6833/1144032959] up:boot 2024-09-13T00:32:31.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[22596]: cluster 2024-09-13T00:32:30.875114+0000 mon.a (mon.0) 778 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi088.krmowh=up:replay} 2024-09-13T00:32:31.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:31 smithi088 bash[24895]: cluster 2024-09-13T00:32:29.803408+0000 mgr.y (mgr.14397) 130 : cluster [DBG] pgmap v105: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:31.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:31 smithi088 bash[24895]: cluster 2024-09-13T00:32:30.180609+0000 mon.a (mon.0) 770 : cluster [DBG] mds.? [v2:172.21.15.77:6834/506597872,v1:172.21.15.77:6835/506597872] up:active 2024-09-13T00:32:31.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:31 smithi088 bash[24895]: cluster 2024-09-13T00:32:30.180724+0000 mon.a (mon.0) 771 : cluster [DBG] fsmap foo:1 {0=foo.smithi077.bigbbf=up:active} 2024-09-13T00:32:31.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:31 smithi088 bash[24895]: cluster 2024-09-13T00:32:30.861576+0000 mon.a (mon.0) 772 : cluster [INF] Dropping low affinity active daemon mds.foo.smithi077.bigbbf in favor of higher affinity standby. 2024-09-13T00:32:31.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:31 smithi088 bash[24895]: cluster 2024-09-13T00:32:30.861642+0000 mon.a (mon.0) 773 : cluster [WRN] Replacing daemon mds.foo.smithi077.bigbbf as rank 0 with standby daemon mds.foo.smithi088.krmowh 2024-09-13T00:32:31.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:31 smithi088 bash[24895]: cluster 2024-09-13T00:32:30.862219+0000 mon.a (mon.0) 774 : cluster [WRN] Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-09-13T00:32:31.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:31 smithi088 bash[24895]: cluster 2024-09-13T00:32:30.862258+0000 mon.a (mon.0) 775 : cluster [WRN] Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-09-13T00:32:31.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:31 smithi088 bash[24895]: audit 2024-09-13T00:32:30.873990+0000 mon.c (mon.2) 92 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi088.krmowh"}]: dispatch 2024-09-13T00:32:31.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:31 smithi088 bash[24895]: cluster 2024-09-13T00:32:30.874538+0000 mon.a (mon.0) 776 : cluster [DBG] osdmap e63: 8 total, 8 up, 8 in 2024-09-13T00:32:31.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:31 smithi088 bash[24895]: cluster 2024-09-13T00:32:30.874971+0000 mon.a (mon.0) 777 : cluster [DBG] mds.? [v2:172.21.15.88:6832/1144032959,v1:172.21.15.88:6833/1144032959] up:boot 2024-09-13T00:32:31.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:31 smithi088 bash[24895]: cluster 2024-09-13T00:32:30.875114+0000 mon.a (mon.0) 778 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi088.krmowh=up:replay} 2024-09-13T00:32:31.755 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:32:31 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:32:31] "GET /metrics HTTP/1.1" 200 208860 "" "Prometheus/2.43.0" 2024-09-13T00:32:32.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:32 smithi077 bash[22596]: cluster 2024-09-13T00:32:31.866275+0000 mon.a (mon.0) 779 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T00:32:32.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:32 smithi077 bash[22596]: cluster 2024-09-13T00:32:31.870630+0000 mon.a (mon.0) 780 : cluster [DBG] mds.? [v2:172.21.15.88:6832/1144032959,v1:172.21.15.88:6833/1144032959] up:reconnect 2024-09-13T00:32:32.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:32 smithi077 bash[22596]: cluster 2024-09-13T00:32:31.870679+0000 mon.a (mon.0) 781 : cluster [DBG] mds.? [v2:172.21.15.77:6834/3765041927,v1:172.21.15.77:6835/3765041927] up:boot 2024-09-13T00:32:32.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:32 smithi077 bash[22596]: cluster 2024-09-13T00:32:31.870743+0000 mon.a (mon.0) 782 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi088.krmowh=up:reconnect} 1 up:standby 2024-09-13T00:32:32.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:32 smithi077 bash[22596]: audit 2024-09-13T00:32:31.870880+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi077.bigbbf"}]: dispatch 2024-09-13T00:32:32.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:32 smithi077 bash[32040]: cluster 2024-09-13T00:32:31.866275+0000 mon.a (mon.0) 779 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T00:32:32.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:32 smithi077 bash[32040]: cluster 2024-09-13T00:32:31.870630+0000 mon.a (mon.0) 780 : cluster [DBG] mds.? [v2:172.21.15.88:6832/1144032959,v1:172.21.15.88:6833/1144032959] up:reconnect 2024-09-13T00:32:32.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:32 smithi077 bash[32040]: cluster 2024-09-13T00:32:31.870679+0000 mon.a (mon.0) 781 : cluster [DBG] mds.? [v2:172.21.15.77:6834/3765041927,v1:172.21.15.77:6835/3765041927] up:boot 2024-09-13T00:32:32.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:32 smithi077 bash[32040]: cluster 2024-09-13T00:32:31.870743+0000 mon.a (mon.0) 782 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi088.krmowh=up:reconnect} 1 up:standby 2024-09-13T00:32:32.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:32 smithi077 bash[32040]: audit 2024-09-13T00:32:31.870880+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi077.bigbbf"}]: dispatch 2024-09-13T00:32:32.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:32 smithi088 bash[24895]: cluster 2024-09-13T00:32:31.866275+0000 mon.a (mon.0) 779 : cluster [INF] Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-09-13T00:32:32.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:32 smithi088 bash[24895]: cluster 2024-09-13T00:32:31.870630+0000 mon.a (mon.0) 780 : cluster [DBG] mds.? [v2:172.21.15.88:6832/1144032959,v1:172.21.15.88:6833/1144032959] up:reconnect 2024-09-13T00:32:32.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:32 smithi088 bash[24895]: cluster 2024-09-13T00:32:31.870679+0000 mon.a (mon.0) 781 : cluster [DBG] mds.? [v2:172.21.15.77:6834/3765041927,v1:172.21.15.77:6835/3765041927] up:boot 2024-09-13T00:32:32.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:32 smithi088 bash[24895]: cluster 2024-09-13T00:32:31.870743+0000 mon.a (mon.0) 782 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi088.krmowh=up:reconnect} 1 up:standby 2024-09-13T00:32:32.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:32 smithi088 bash[24895]: audit 2024-09-13T00:32:31.870880+0000 mon.c (mon.2) 93 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi077.bigbbf"}]: dispatch 2024-09-13T00:32:33.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:33 smithi077 bash[32040]: cluster 2024-09-13T00:32:31.803811+0000 mgr.y (mgr.14397) 131 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:33.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:33 smithi077 bash[32040]: cluster 2024-09-13T00:32:32.875183+0000 mon.a (mon.0) 783 : cluster [DBG] mds.? [v2:172.21.15.88:6832/1144032959,v1:172.21.15.88:6833/1144032959] up:rejoin 2024-09-13T00:32:33.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:33 smithi077 bash[32040]: cluster 2024-09-13T00:32:32.875306+0000 mon.a (mon.0) 784 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi088.krmowh=up:rejoin} 1 up:standby 2024-09-13T00:32:33.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:33 smithi077 bash[32040]: cluster 2024-09-13T00:32:32.875990+0000 mon.a (mon.0) 785 : cluster [INF] daemon mds.foo.smithi088.krmowh is now active in filesystem foo as rank 0 2024-09-13T00:32:33.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:33 smithi077 bash[32040]: audit 2024-09-13T00:32:32.948075+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:33.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:33 smithi077 bash[22596]: cluster 2024-09-13T00:32:31.803811+0000 mgr.y (mgr.14397) 131 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:33.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:33 smithi077 bash[22596]: cluster 2024-09-13T00:32:32.875183+0000 mon.a (mon.0) 783 : cluster [DBG] mds.? [v2:172.21.15.88:6832/1144032959,v1:172.21.15.88:6833/1144032959] up:rejoin 2024-09-13T00:32:33.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:33 smithi077 bash[22596]: cluster 2024-09-13T00:32:32.875306+0000 mon.a (mon.0) 784 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi088.krmowh=up:rejoin} 1 up:standby 2024-09-13T00:32:33.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:33 smithi077 bash[22596]: cluster 2024-09-13T00:32:32.875990+0000 mon.a (mon.0) 785 : cluster [INF] daemon mds.foo.smithi088.krmowh is now active in filesystem foo as rank 0 2024-09-13T00:32:33.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:33 smithi077 bash[22596]: audit 2024-09-13T00:32:32.948075+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:33.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:33 smithi088 bash[24895]: cluster 2024-09-13T00:32:31.803811+0000 mgr.y (mgr.14397) 131 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:33.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:33 smithi088 bash[24895]: cluster 2024-09-13T00:32:32.875183+0000 mon.a (mon.0) 783 : cluster [DBG] mds.? [v2:172.21.15.88:6832/1144032959,v1:172.21.15.88:6833/1144032959] up:rejoin 2024-09-13T00:32:33.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:33 smithi088 bash[24895]: cluster 2024-09-13T00:32:32.875306+0000 mon.a (mon.0) 784 : cluster [DBG] fsmap foo:1/1 {0=foo.smithi088.krmowh=up:rejoin} 1 up:standby 2024-09-13T00:32:33.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:33 smithi088 bash[24895]: cluster 2024-09-13T00:32:32.875990+0000 mon.a (mon.0) 785 : cluster [INF] daemon mds.foo.smithi088.krmowh is now active in filesystem foo as rank 0 2024-09-13T00:32:33.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:33 smithi088 bash[24895]: audit 2024-09-13T00:32:32.948075+0000 mon.a (mon.0) 786 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:34.398 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:34 smithi077 bash[32040]: cluster 2024-09-13T00:32:33.870169+0000 mon.a (mon.0) 787 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T00:32:34.398 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:34 smithi077 bash[32040]: cluster 2024-09-13T00:32:33.870201+0000 mon.a (mon.0) 788 : cluster [INF] Cluster is now healthy 2024-09-13T00:32:34.398 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:34 smithi077 bash[32040]: cluster 2024-09-13T00:32:33.874662+0000 mon.a (mon.0) 789 : cluster [DBG] mds.? [v2:172.21.15.88:6832/1144032959,v1:172.21.15.88:6833/1144032959] up:active 2024-09-13T00:32:34.398 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:34 smithi077 bash[32040]: cluster 2024-09-13T00:32:33.874731+0000 mon.a (mon.0) 790 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:32:34.398 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:34 smithi077 bash[32040]: audit 2024-09-13T00:32:34.022430+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:34.398 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:34 smithi077 bash[32040]: audit 2024-09-13T00:32:34.027749+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:34.398 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:34 smithi077 bash[22596]: cluster 2024-09-13T00:32:33.870169+0000 mon.a (mon.0) 787 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T00:32:34.399 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:34 smithi077 bash[22596]: cluster 2024-09-13T00:32:33.870201+0000 mon.a (mon.0) 788 : cluster [INF] Cluster is now healthy 2024-09-13T00:32:34.399 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:34 smithi077 bash[22596]: cluster 2024-09-13T00:32:33.874662+0000 mon.a (mon.0) 789 : cluster [DBG] mds.? [v2:172.21.15.88:6832/1144032959,v1:172.21.15.88:6833/1144032959] up:active 2024-09-13T00:32:34.399 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:34 smithi077 bash[22596]: cluster 2024-09-13T00:32:33.874731+0000 mon.a (mon.0) 790 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:32:34.399 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:34 smithi077 bash[22596]: audit 2024-09-13T00:32:34.022430+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:34.399 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:34 smithi077 bash[22596]: audit 2024-09-13T00:32:34.027749+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:34.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:34 smithi088 bash[24895]: cluster 2024-09-13T00:32:33.870169+0000 mon.a (mon.0) 787 : cluster [INF] Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-09-13T00:32:34.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:34 smithi088 bash[24895]: cluster 2024-09-13T00:32:33.870201+0000 mon.a (mon.0) 788 : cluster [INF] Cluster is now healthy 2024-09-13T00:32:34.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:34 smithi088 bash[24895]: cluster 2024-09-13T00:32:33.874662+0000 mon.a (mon.0) 789 : cluster [DBG] mds.? [v2:172.21.15.88:6832/1144032959,v1:172.21.15.88:6833/1144032959] up:active 2024-09-13T00:32:34.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:34 smithi088 bash[24895]: cluster 2024-09-13T00:32:33.874731+0000 mon.a (mon.0) 790 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:32:34.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:34 smithi088 bash[24895]: audit 2024-09-13T00:32:34.022430+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:34.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:34 smithi088 bash[24895]: audit 2024-09-13T00:32:34.027749+0000 mon.a (mon.0) 792 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:35.149 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-13T00:32:35.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[22596]: cluster 2024-09-13T00:32:33.804313+0000 mgr.y (mgr.14397) 132 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-13T00:32:35.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[22596]: audit 2024-09-13T00:32:35.117552+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:35.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[22596]: audit 2024-09-13T00:32:35.130011+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:35.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[22596]: audit 2024-09-13T00:32:35.132470+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:35.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[22596]: audit 2024-09-13T00:32:35.133851+0000 mon.c (mon.2) 95 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:32:35.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[22596]: audit 2024-09-13T00:32:35.141772+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:35.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[22596]: audit 2024-09-13T00:32:35.190645+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:35.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[22596]: audit 2024-09-13T00:32:35.192409+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:35.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[22596]: audit 2024-09-13T00:32:35.193246+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:32:35.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[22596]: audit 2024-09-13T00:32:35.199900+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:35.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[32040]: cluster 2024-09-13T00:32:33.804313+0000 mgr.y (mgr.14397) 132 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-13T00:32:35.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[32040]: audit 2024-09-13T00:32:35.117552+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:35.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[32040]: audit 2024-09-13T00:32:35.130011+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:35.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[32040]: audit 2024-09-13T00:32:35.132470+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:35.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[32040]: audit 2024-09-13T00:32:35.133851+0000 mon.c (mon.2) 95 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:32:35.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[32040]: audit 2024-09-13T00:32:35.141772+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:35.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[32040]: audit 2024-09-13T00:32:35.190645+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:35.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[32040]: audit 2024-09-13T00:32:35.192409+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:35.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[32040]: audit 2024-09-13T00:32:35.193246+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:32:35.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:35 smithi077 bash[32040]: audit 2024-09-13T00:32:35.199900+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:35.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:35 smithi088 bash[24895]: cluster 2024-09-13T00:32:33.804313+0000 mgr.y (mgr.14397) 132 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-13T00:32:35.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:35 smithi088 bash[24895]: audit 2024-09-13T00:32:35.117552+0000 mon.a (mon.0) 793 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:35.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:35 smithi088 bash[24895]: audit 2024-09-13T00:32:35.130011+0000 mon.a (mon.0) 794 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:35.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:35 smithi088 bash[24895]: audit 2024-09-13T00:32:35.132470+0000 mon.c (mon.2) 94 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:35.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:35 smithi088 bash[24895]: audit 2024-09-13T00:32:35.133851+0000 mon.c (mon.2) 95 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:32:35.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:35 smithi088 bash[24895]: audit 2024-09-13T00:32:35.141772+0000 mon.a (mon.0) 795 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:35.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:35 smithi088 bash[24895]: audit 2024-09-13T00:32:35.190645+0000 mon.c (mon.2) 96 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:32:35.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:35 smithi088 bash[24895]: audit 2024-09-13T00:32:35.192409+0000 mon.c (mon.2) 97 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:32:35.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:35 smithi088 bash[24895]: audit 2024-09-13T00:32:35.193246+0000 mon.c (mon.2) 98 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:32:35.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:35 smithi088 bash[24895]: audit 2024-09-13T00:32:35.199900+0000 mon.a (mon.0) 796 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:37.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:37 smithi077 bash[22596]: cluster 2024-09-13T00:32:35.804825+0000 mgr.y (mgr.14397) 133 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-13T00:32:37.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:37 smithi077 bash[32040]: cluster 2024-09-13T00:32:35.804825+0000 mgr.y (mgr.14397) 133 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-13T00:32:37.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:37 smithi088 bash[24895]: cluster 2024-09-13T00:32:35.804825+0000 mgr.y (mgr.14397) 133 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-13T00:32:39.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:39 smithi077 bash[22596]: cluster 2024-09-13T00:32:37.805398+0000 mgr.y (mgr.14397) 134 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-13T00:32:39.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:39 smithi077 bash[32040]: cluster 2024-09-13T00:32:37.805398+0000 mgr.y (mgr.14397) 134 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-13T00:32:39.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:39 smithi088 bash[24895]: cluster 2024-09-13T00:32:37.805398+0000 mgr.y (mgr.14397) 134 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 1.3 KiB/s wr, 5 op/s 2024-09-13T00:32:39.927 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:32:41.099 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:32:40 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:32:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:32:41.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:41 smithi077 bash[22596]: cluster 2024-09-13T00:32:39.806481+0000 mgr.y (mgr.14397) 135 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-09-13T00:32:41.460 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:41 smithi077 bash[32040]: cluster 2024-09-13T00:32:39.806481+0000 mgr.y (mgr.14397) 135 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-09-13T00:32:41.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:41 smithi088 bash[24895]: cluster 2024-09-13T00:32:39.806481+0000 mgr.y (mgr.14397) 135 : cluster [DBG] pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 9 op/s 2024-09-13T00:32:41.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:32:41 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:32:41] "GET /metrics HTTP/1.1" 200 208860 "" "Prometheus/2.43.0" 2024-09-13T00:32:43.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:43 smithi077 bash[22596]: cluster 2024-09-13T00:32:41.807092+0000 mgr.y (mgr.14397) 136 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-09-13T00:32:43.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:43 smithi077 bash[22596]: audit 2024-09-13T00:32:42.838860+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:43.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:43 smithi077 bash[22596]: audit 2024-09-13T00:32:42.840865+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:43.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:43 smithi077 bash[22596]: audit 2024-09-13T00:32:42.964374+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:43.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:43 smithi077 bash[32040]: cluster 2024-09-13T00:32:41.807092+0000 mgr.y (mgr.14397) 136 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-09-13T00:32:43.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:43 smithi077 bash[32040]: audit 2024-09-13T00:32:42.838860+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:43.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:43 smithi077 bash[32040]: audit 2024-09-13T00:32:42.840865+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:43.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:43 smithi077 bash[32040]: audit 2024-09-13T00:32:42.964374+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:43.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:43 smithi088 bash[24895]: cluster 2024-09-13T00:32:41.807092+0000 mgr.y (mgr.14397) 136 : cluster [DBG] pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-09-13T00:32:43.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:43 smithi088 bash[24895]: audit 2024-09-13T00:32:42.838860+0000 mon.a (mon.0) 797 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:43.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:43 smithi088 bash[24895]: audit 2024-09-13T00:32:42.840865+0000 mon.c (mon.2) 99 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:43.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:43 smithi088 bash[24895]: audit 2024-09-13T00:32:42.964374+0000 mon.a (mon.0) 798 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:32:44.530 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-13T00:32:44.542 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T00:32:44.542 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T00:32:44.542 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T00:32:45.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:45 smithi077 bash[22596]: cluster 2024-09-13T00:32:43.808055+0000 mgr.y (mgr.14397) 137 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 KiB/s rd, 1.1 KiB/s wr, 7 op/s 2024-09-13T00:32:45.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:45 smithi077 bash[32040]: cluster 2024-09-13T00:32:43.808055+0000 mgr.y (mgr.14397) 137 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 KiB/s rd, 1.1 KiB/s wr, 7 op/s 2024-09-13T00:32:45.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:45 smithi088 bash[24895]: cluster 2024-09-13T00:32:43.808055+0000 mgr.y (mgr.14397) 137 : cluster [DBG] pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 4.1 KiB/s rd, 1.1 KiB/s wr, 7 op/s 2024-09-13T00:32:47.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:47 smithi077 bash[22596]: cluster 2024-09-13T00:32:45.808674+0000 mgr.y (mgr.14397) 138 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T00:32:47.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:47 smithi077 bash[32040]: cluster 2024-09-13T00:32:45.808674+0000 mgr.y (mgr.14397) 138 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T00:32:47.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:47 smithi088 bash[24895]: cluster 2024-09-13T00:32:45.808674+0000 mgr.y (mgr.14397) 138 : cluster [DBG] pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T00:32:49.322 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:32:49.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:49 smithi077 bash[22596]: cluster 2024-09-13T00:32:47.809255+0000 mgr.y (mgr.14397) 139 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T00:32:49.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:49 smithi077 bash[32040]: cluster 2024-09-13T00:32:47.809255+0000 mgr.y (mgr.14397) 139 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T00:32:49.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:49 smithi088 bash[24895]: cluster 2024-09-13T00:32:47.809255+0000 mgr.y (mgr.14397) 139 : cluster [DBG] pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T00:32:51.099 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:32:50 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:32:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:32:51.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:51 smithi077 bash[22596]: cluster 2024-09-13T00:32:49.810357+0000 mgr.y (mgr.14397) 140 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T00:32:51.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:51 smithi077 bash[32040]: cluster 2024-09-13T00:32:49.810357+0000 mgr.y (mgr.14397) 140 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T00:32:51.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:51 smithi088 bash[24895]: cluster 2024-09-13T00:32:49.810357+0000 mgr.y (mgr.14397) 140 : cluster [DBG] pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-13T00:32:51.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:32:51 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:32:51] "GET /metrics HTTP/1.1" 200 240053 "" "Prometheus/2.43.0" 2024-09-13T00:32:53.144 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:32:53.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:53 smithi077 bash[22596]: cluster 2024-09-13T00:32:51.811008+0000 mgr.y (mgr.14397) 141 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:53.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:53 smithi077 bash[22596]: audit 2024-09-13T00:32:53.143071+0000 mon.a (mon.0) 799 : audit [DBG] from='client.? 172.21.15.77:0/2927698753' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T00:32:53.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:53 smithi077 bash[32040]: cluster 2024-09-13T00:32:51.811008+0000 mgr.y (mgr.14397) 141 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:53.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:53 smithi077 bash[32040]: audit 2024-09-13T00:32:53.143071+0000 mon.a (mon.0) 799 : audit [DBG] from='client.? 172.21.15.77:0/2927698753' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T00:32:53.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:53 smithi088 bash[24895]: cluster 2024-09-13T00:32:51.811008+0000 mgr.y (mgr.14397) 141 : cluster [DBG] pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:53.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:53 smithi088 bash[24895]: audit 2024-09-13T00:32:53.143071+0000 mon.a (mon.0) 799 : audit [DBG] from='client.? 172.21.15.77:0/2927698753' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T00:32:53.956 INFO:teuthology.orchestra.run.smithi077.stdout:{"epoch":20,"flags":0,"active_gid":14397,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6800","nonce":930711876},{"type":"v1","addr":"172.21.15.77:6801","nonce":930711876}]},"active_addr":"172.21.15.77:6801/930711876","active_change":"2024-09-13T00:29:27.728667+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24289,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.77:8443/","prometheus":"http://172.21.15.77:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":51,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.77:0","nonce":1115799687}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.77:0","nonce":3515285019}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.77:0","nonce":2219740569}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.77:0","nonce":676410089}]}]} 2024-09-13T00:32:53.960 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T00:32:53.961 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T00:32:53.961 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T00:32:55.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:55 smithi077 bash[22596]: cluster 2024-09-13T00:32:53.812033+0000 mgr.y (mgr.14397) 142 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:55.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:55 smithi077 bash[32040]: cluster 2024-09-13T00:32:53.812033+0000 mgr.y (mgr.14397) 142 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:55.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:55 smithi088 bash[24895]: cluster 2024-09-13T00:32:53.812033+0000 mgr.y (mgr.14397) 142 : cluster [DBG] pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:57.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:57 smithi077 bash[22596]: cluster 2024-09-13T00:32:55.812675+0000 mgr.y (mgr.14397) 143 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:57.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:57 smithi077 bash[32040]: cluster 2024-09-13T00:32:55.812675+0000 mgr.y (mgr.14397) 143 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:57.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:57 smithi088 bash[24895]: cluster 2024-09-13T00:32:55.812675+0000 mgr.y (mgr.14397) 143 : cluster [DBG] pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:58.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:58 smithi077 bash[22596]: audit 2024-09-13T00:32:57.833371+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:58.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:58 smithi077 bash[32040]: audit 2024-09-13T00:32:57.833371+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:58.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:58 smithi088 bash[24895]: audit 2024-09-13T00:32:57.833371+0000 mon.c (mon.2) 100 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:32:58.753 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:32:59.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:32:59 smithi077 bash[22596]: cluster 2024-09-13T00:32:57.813220+0000 mgr.y (mgr.14397) 144 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:59.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:32:59 smithi077 bash[32040]: cluster 2024-09-13T00:32:57.813220+0000 mgr.y (mgr.14397) 144 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:32:59.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:32:59 smithi088 bash[24895]: cluster 2024-09-13T00:32:57.813220+0000 mgr.y (mgr.14397) 144 : cluster [DBG] pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:01.099 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:33:00 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:33:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:33:01.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:01 smithi077 bash[22596]: cluster 2024-09-13T00:32:59.814289+0000 mgr.y (mgr.14397) 145 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:01.525 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:33:01 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:33:01] "GET /metrics HTTP/1.1" 200 240055 "" "Prometheus/2.43.0" 2024-09-13T00:33:01.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:01 smithi077 bash[32040]: cluster 2024-09-13T00:32:59.814289+0000 mgr.y (mgr.14397) 145 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:01.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:01 smithi088 bash[24895]: cluster 2024-09-13T00:32:59.814289+0000 mgr.y (mgr.14397) 145 : cluster [DBG] pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:02.695 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:33:02.696 INFO:teuthology.orchestra.run.smithi077.stdout:{"epoch":63,"fsid":"9218306c-7165-11ef-bceb-c7b262605968","created":"2024-09-13T00:18:59.375829+0000","modified":"2024-09-13T00:32:30.861742+0000","last_up_change":"2024-09-13T00:28:29.926728+0000","last_in_change":"2024-09-13T00:28:05.097746+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T00:24:15.726444+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T00:30:09.880408+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":57,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T00:32:21.904347+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T00:32:22.489687+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"e0860feb-72c9-4c49-933d-0a49b68caeee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6802","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6803","nonce":1623816886}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6804","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6805","nonce":1623816886}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6808","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6809","nonce":1623816886}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6806","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6807","nonce":1623816886}]},"public_addr":"172.21.15.77:6803/1623816886","cluster_addr":"172.21.15.77:6805/1623816886","heartbeat_back_addr":"172.21.15.77:6809/1623816886","heartbeat_front_addr":"172.21.15.77:6807/1623816886","state":["exists","up"]},{"osd":1,"uuid":"7ee01388-61b0-4a60-81f2-08c3c072ec0e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6810","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6811","nonce":1691613782}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6812","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6813","nonce":1691613782}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6816","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6817","nonce":1691613782}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6814","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6815","nonce":1691613782}]},"public_addr":"172.21.15.77:6811/1691613782","cluster_addr":"172.21.15.77:6813/1691613782","heartbeat_back_addr":"172.21.15.77:6817/1691613782","heartbeat_front_addr":"172.21.15.77:6815/1691613782","state":["exists","up"]},{"osd":2,"uuid":"ff8bc88a-5b11-469f-912d-d3149036b450","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6818","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6819","nonce":2379210664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6820","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6821","nonce":2379210664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6824","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6825","nonce":2379210664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6822","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6823","nonce":2379210664}]},"public_addr":"172.21.15.77:6819/2379210664","cluster_addr":"172.21.15.77:6821/2379210664","heartbeat_back_addr":"172.21.15.77:6825/2379210664","heartbeat_front_addr":"172.21.15.77:6823/2379210664","state":["exists","up"]},{"osd":3,"uuid":"cb032137-609e-41f6-bfc7-e70b77ea9456","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6826","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6827","nonce":2636296765}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6828","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6829","nonce":2636296765}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6832","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6833","nonce":2636296765}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6830","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6831","nonce":2636296765}]},"public_addr":"172.21.15.77:6827/2636296765","cluster_addr":"172.21.15.77:6829/2636296765","heartbeat_back_addr":"172.21.15.77:6833/2636296765","heartbeat_front_addr":"172.21.15.77:6831/2636296765","state":["exists","up"]},{"osd":4,"uuid":"cd38603e-5cdc-42b6-b8c8-656d44df0ed5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6800","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6801","nonce":3645723145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6802","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6803","nonce":3645723145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6806","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6807","nonce":3645723145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6804","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6805","nonce":3645723145}]},"public_addr":"172.21.15.88:6801/3645723145","cluster_addr":"172.21.15.88:6803/3645723145","heartbeat_back_addr":"172.21.15.88:6807/3645723145","heartbeat_front_addr":"172.21.15.88:6805/3645723145","state":["exists","up"]},{"osd":5,"uuid":"d4b81ff3-f257-4f63-a7a7-cbd1ff754544","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6808","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6809","nonce":2699666173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6810","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6811","nonce":2699666173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6814","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6815","nonce":2699666173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6812","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6813","nonce":2699666173}]},"public_addr":"172.21.15.88:6809/2699666173","cluster_addr":"172.21.15.88:6811/2699666173","heartbeat_back_addr":"172.21.15.88:6815/2699666173","heartbeat_front_addr":"172.21.15.88:6813/2699666173","state":["exists","up"]},{"osd":6,"uuid":"31c0d169-c329-440d-848b-057fbbf418ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6816","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6817","nonce":412460705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6818","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6819","nonce":412460705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6822","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6823","nonce":412460705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6820","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6821","nonce":412460705}]},"public_addr":"172.21.15.88:6817/412460705","cluster_addr":"172.21.15.88:6819/412460705","heartbeat_back_addr":"172.21.15.88:6823/412460705","heartbeat_front_addr":"172.21.15.88:6821/412460705","state":["exists","up"]},{"osd":7,"uuid":"2b2c1a7c-3203-4e9a-b07b-4eaae7e09c0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6824","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6825","nonce":2009347492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6826","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6827","nonce":2009347492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6830","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6831","nonce":2009347492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6828","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6829","nonce":2009347492}]},"public_addr":"172.21.15.88:6825/2009347492","cluster_addr":"172.21.15.88:6827/2009347492","heartbeat_back_addr":"172.21.15.88:6831/2009347492","heartbeat_front_addr":"172.21.15.88:6829/2009347492","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:22:21.643909+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:23:17.132405+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:24:13.726010+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:25:11.211228+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:26:00.949032+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:26:49.596019+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:27:38.694576+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:28:28.383054+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.77:6835/506597872":"2024-09-14T00:32:30.861677+0000","172.21.15.77:6834/506597872":"2024-09-14T00:32:30.861677+0000","172.21.15.77:0/1008984793":"2024-09-14T00:29:27.728349+0000","172.21.15.77:6801/395417143":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/892433806":"2024-09-14T00:19:25.064947+0000","172.21.15.77:0/4291417054":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6800/426595304":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6801/426595304":"2024-09-14T00:19:25.064947+0000","172.21.15.77:0/1137737372":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6800/395417143":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/1897411349":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/4117010497":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/2198980039":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/1347196058":"2024-09-14T00:29:27.728349+0000","172.21.15.77:6800/3213372405":"2024-09-14T00:19:55.144762+0000","172.21.15.77:6801/3213372405":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/1177255174":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/3827884320":"2024-09-14T00:29:27.728349+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T00:33:03.459 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T00:33:03.460 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T00:33:03.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:03 smithi077 bash[22596]: cluster 2024-09-13T00:33:01.814904+0000 mgr.y (mgr.14397) 146 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:03.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:03 smithi077 bash[22596]: audit 2024-09-13T00:33:02.700644+0000 mon.c (mon.2) 101 : audit [DBG] from='client.? 172.21.15.77:0/3485545429' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:33:03.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:03 smithi077 bash[32040]: cluster 2024-09-13T00:33:01.814904+0000 mgr.y (mgr.14397) 146 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:03.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:03 smithi077 bash[32040]: audit 2024-09-13T00:33:02.700644+0000 mon.c (mon.2) 101 : audit [DBG] from='client.? 172.21.15.77:0/3485545429' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:33:03.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:03 smithi088 bash[24895]: cluster 2024-09-13T00:33:01.814904+0000 mgr.y (mgr.14397) 146 : cluster [DBG] pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:03.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:03 smithi088 bash[24895]: audit 2024-09-13T00:33:02.700644+0000 mon.c (mon.2) 101 : audit [DBG] from='client.? 172.21.15.77:0/3485545429' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:33:05.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:05 smithi088 bash[24895]: cluster 2024-09-13T00:33:03.815904+0000 mgr.y (mgr.14397) 147 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:05.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:05 smithi077 bash[22596]: cluster 2024-09-13T00:33:03.815904+0000 mgr.y (mgr.14397) 147 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:05.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:05 smithi077 bash[32040]: cluster 2024-09-13T00:33:03.815904+0000 mgr.y (mgr.14397) 147 : cluster [DBG] pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:07.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:07 smithi088 bash[24895]: cluster 2024-09-13T00:33:05.816506+0000 mgr.y (mgr.14397) 148 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:07.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:07 smithi077 bash[22596]: cluster 2024-09-13T00:33:05.816506+0000 mgr.y (mgr.14397) 148 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:07.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:07 smithi077 bash[32040]: cluster 2024-09-13T00:33:05.816506+0000 mgr.y (mgr.14397) 148 : cluster [DBG] pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:08.244 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:09.592 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:09 smithi077 bash[22596]: cluster 2024-09-13T00:33:07.817103+0000 mgr.y (mgr.14397) 149 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:09.592 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:09 smithi077 bash[32040]: cluster 2024-09-13T00:33:07.817103+0000 mgr.y (mgr.14397) 149 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:09.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:09 smithi088 bash[24895]: cluster 2024-09-13T00:33:07.817103+0000 mgr.y (mgr.14397) 149 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:11.099 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:33:10 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:33:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:33:11.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:11 smithi088 bash[24895]: cluster 2024-09-13T00:33:09.818207+0000 mgr.y (mgr.14397) 150 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:11.775 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:33:11.776 INFO:teuthology.orchestra.run.smithi077.stdout:{"epoch":63,"fsid":"9218306c-7165-11ef-bceb-c7b262605968","created":"2024-09-13T00:18:59.375829+0000","modified":"2024-09-13T00:32:30.861742+0000","last_up_change":"2024-09-13T00:28:29.926728+0000","last_in_change":"2024-09-13T00:28:05.097746+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T00:24:15.726444+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T00:30:09.880408+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":57,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T00:32:21.904347+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T00:32:22.489687+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"e0860feb-72c9-4c49-933d-0a49b68caeee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6802","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6803","nonce":1623816886}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6804","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6805","nonce":1623816886}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6808","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6809","nonce":1623816886}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6806","nonce":1623816886},{"type":"v1","addr":"172.21.15.77:6807","nonce":1623816886}]},"public_addr":"172.21.15.77:6803/1623816886","cluster_addr":"172.21.15.77:6805/1623816886","heartbeat_back_addr":"172.21.15.77:6809/1623816886","heartbeat_front_addr":"172.21.15.77:6807/1623816886","state":["exists","up"]},{"osd":1,"uuid":"7ee01388-61b0-4a60-81f2-08c3c072ec0e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6810","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6811","nonce":1691613782}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6812","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6813","nonce":1691613782}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6816","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6817","nonce":1691613782}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6814","nonce":1691613782},{"type":"v1","addr":"172.21.15.77:6815","nonce":1691613782}]},"public_addr":"172.21.15.77:6811/1691613782","cluster_addr":"172.21.15.77:6813/1691613782","heartbeat_back_addr":"172.21.15.77:6817/1691613782","heartbeat_front_addr":"172.21.15.77:6815/1691613782","state":["exists","up"]},{"osd":2,"uuid":"ff8bc88a-5b11-469f-912d-d3149036b450","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6818","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6819","nonce":2379210664}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6820","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6821","nonce":2379210664}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6824","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6825","nonce":2379210664}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6822","nonce":2379210664},{"type":"v1","addr":"172.21.15.77:6823","nonce":2379210664}]},"public_addr":"172.21.15.77:6819/2379210664","cluster_addr":"172.21.15.77:6821/2379210664","heartbeat_back_addr":"172.21.15.77:6825/2379210664","heartbeat_front_addr":"172.21.15.77:6823/2379210664","state":["exists","up"]},{"osd":3,"uuid":"cb032137-609e-41f6-bfc7-e70b77ea9456","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6826","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6827","nonce":2636296765}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6828","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6829","nonce":2636296765}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6832","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6833","nonce":2636296765}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6830","nonce":2636296765},{"type":"v1","addr":"172.21.15.77:6831","nonce":2636296765}]},"public_addr":"172.21.15.77:6827/2636296765","cluster_addr":"172.21.15.77:6829/2636296765","heartbeat_back_addr":"172.21.15.77:6833/2636296765","heartbeat_front_addr":"172.21.15.77:6831/2636296765","state":["exists","up"]},{"osd":4,"uuid":"cd38603e-5cdc-42b6-b8c8-656d44df0ed5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6800","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6801","nonce":3645723145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6802","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6803","nonce":3645723145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6806","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6807","nonce":3645723145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6804","nonce":3645723145},{"type":"v1","addr":"172.21.15.88:6805","nonce":3645723145}]},"public_addr":"172.21.15.88:6801/3645723145","cluster_addr":"172.21.15.88:6803/3645723145","heartbeat_back_addr":"172.21.15.88:6807/3645723145","heartbeat_front_addr":"172.21.15.88:6805/3645723145","state":["exists","up"]},{"osd":5,"uuid":"d4b81ff3-f257-4f63-a7a7-cbd1ff754544","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6808","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6809","nonce":2699666173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6810","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6811","nonce":2699666173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6814","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6815","nonce":2699666173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6812","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6813","nonce":2699666173}]},"public_addr":"172.21.15.88:6809/2699666173","cluster_addr":"172.21.15.88:6811/2699666173","heartbeat_back_addr":"172.21.15.88:6815/2699666173","heartbeat_front_addr":"172.21.15.88:6813/2699666173","state":["exists","up"]},{"osd":6,"uuid":"31c0d169-c329-440d-848b-057fbbf418ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6816","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6817","nonce":412460705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6818","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6819","nonce":412460705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6822","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6823","nonce":412460705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6820","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6821","nonce":412460705}]},"public_addr":"172.21.15.88:6817/412460705","cluster_addr":"172.21.15.88:6819/412460705","heartbeat_back_addr":"172.21.15.88:6823/412460705","heartbeat_front_addr":"172.21.15.88:6821/412460705","state":["exists","up"]},{"osd":7,"uuid":"2b2c1a7c-3203-4e9a-b07b-4eaae7e09c0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":59,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6824","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6825","nonce":2009347492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6826","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6827","nonce":2009347492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6830","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6831","nonce":2009347492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6828","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6829","nonce":2009347492}]},"public_addr":"172.21.15.88:6825/2009347492","cluster_addr":"172.21.15.88:6827/2009347492","heartbeat_back_addr":"172.21.15.88:6831/2009347492","heartbeat_front_addr":"172.21.15.88:6829/2009347492","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:22:21.643909+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:23:17.132405+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:24:13.726010+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:25:11.211228+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:26:00.949032+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:26:49.596019+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:27:38.694576+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:28:28.383054+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.77:6835/506597872":"2024-09-14T00:32:30.861677+0000","172.21.15.77:6834/506597872":"2024-09-14T00:32:30.861677+0000","172.21.15.77:0/1008984793":"2024-09-14T00:29:27.728349+0000","172.21.15.77:6801/395417143":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/892433806":"2024-09-14T00:19:25.064947+0000","172.21.15.77:0/4291417054":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6800/426595304":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6801/426595304":"2024-09-14T00:19:25.064947+0000","172.21.15.77:0/1137737372":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6800/395417143":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/1897411349":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/4117010497":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/2198980039":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/1347196058":"2024-09-14T00:29:27.728349+0000","172.21.15.77:6800/3213372405":"2024-09-14T00:19:55.144762+0000","172.21.15.77:6801/3213372405":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/1177255174":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/3827884320":"2024-09-14T00:29:27.728349+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T00:33:11.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:11 smithi077 bash[22596]: cluster 2024-09-13T00:33:09.818207+0000 mgr.y (mgr.14397) 150 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:11.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:33:11 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:33:11] "GET /metrics HTTP/1.1" 200 240055 "" "Prometheus/2.43.0" 2024-09-13T00:33:11.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:11 smithi077 bash[32040]: cluster 2024-09-13T00:33:09.818207+0000 mgr.y (mgr.14397) 150 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:12.573 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:12 smithi077 bash[22596]: audit 2024-09-13T00:33:11.779238+0000 mon.a (mon.0) 800 : audit [DBG] from='client.? 172.21.15.77:0/2064264567' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:33:12.573 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:12 smithi077 bash[32040]: audit 2024-09-13T00:33:11.779238+0000 mon.a (mon.0) 800 : audit [DBG] from='client.? 172.21.15.77:0/2064264567' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:33:12.575 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T00:33:12.576 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T00:33:12.576 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T00:33:12.577 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T00:33:12.577 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T00:33:12.578 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T00:33:12.578 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T00:33:12.578 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T00:33:12.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:12 smithi088 bash[24895]: audit 2024-09-13T00:33:11.779238+0000 mon.a (mon.0) 800 : audit [DBG] from='client.? 172.21.15.77:0/2064264567' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:33:13.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:13 smithi088 bash[24895]: cluster 2024-09-13T00:33:11.818803+0000 mgr.y (mgr.14397) 151 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:13.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:13 smithi088 bash[24895]: audit 2024-09-13T00:33:12.833086+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:13.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:13 smithi077 bash[32040]: cluster 2024-09-13T00:33:11.818803+0000 mgr.y (mgr.14397) 151 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:13.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:13 smithi077 bash[32040]: audit 2024-09-13T00:33:12.833086+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:13.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:13 smithi077 bash[22596]: cluster 2024-09-13T00:33:11.818803+0000 mgr.y (mgr.14397) 151 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:13.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:13 smithi077 bash[22596]: audit 2024-09-13T00:33:12.833086+0000 mon.c (mon.2) 102 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:15.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:15 smithi088 bash[24895]: cluster 2024-09-13T00:33:13.819793+0000 mgr.y (mgr.14397) 152 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:15.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:15 smithi077 bash[22596]: cluster 2024-09-13T00:33:13.819793+0000 mgr.y (mgr.14397) 152 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:15.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:15 smithi077 bash[32040]: cluster 2024-09-13T00:33:13.819793+0000 mgr.y (mgr.14397) 152 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:17.488 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:17.489 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:17.489 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:17.489 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:17.490 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:17.490 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:17.490 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:17.492 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:17.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:17 smithi088 bash[24895]: cluster 2024-09-13T00:33:15.820362+0000 mgr.y (mgr.14397) 153 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:17.703 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:17 smithi077 bash[32040]: cluster 2024-09-13T00:33:15.820362+0000 mgr.y (mgr.14397) 153 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:17.703 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:17 smithi077 bash[22596]: cluster 2024-09-13T00:33:15.820362+0000 mgr.y (mgr.14397) 153 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:19.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:19 smithi088 bash[24895]: cluster 2024-09-13T00:33:17.821122+0000 mgr.y (mgr.14397) 154 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:19.608 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:19 smithi077 bash[22596]: cluster 2024-09-13T00:33:17.821122+0000 mgr.y (mgr.14397) 154 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:19.608 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:19 smithi077 bash[32040]: cluster 2024-09-13T00:33:17.821122+0000 mgr.y (mgr.14397) 154 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:21.099 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:33:20 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:33:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:33:21.494 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:21 smithi077 bash[22596]: cluster 2024-09-13T00:33:19.822167+0000 mgr.y (mgr.14397) 155 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:21.494 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:21 smithi077 bash[32040]: cluster 2024-09-13T00:33:19.822167+0000 mgr.y (mgr.14397) 155 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:21.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:21 smithi088 bash[24895]: cluster 2024-09-13T00:33:19.822167+0000 mgr.y (mgr.14397) 155 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:21.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:33:21 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:33:21] "GET /metrics HTTP/1.1" 200 240019 "" "Prometheus/2.43.0" 2024-09-13T00:33:23.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:23 smithi088 bash[24895]: cluster 2024-09-13T00:33:21.822930+0000 mgr.y (mgr.14397) 156 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:23.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:23 smithi077 bash[22596]: cluster 2024-09-13T00:33:21.822930+0000 mgr.y (mgr.14397) 156 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:23.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:23 smithi077 bash[32040]: cluster 2024-09-13T00:33:21.822930+0000 mgr.y (mgr.14397) 156 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:25.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:25 smithi088 bash[24895]: cluster 2024-09-13T00:33:23.823983+0000 mgr.y (mgr.14397) 157 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:25.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:25 smithi077 bash[22596]: cluster 2024-09-13T00:33:23.823983+0000 mgr.y (mgr.14397) 157 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:25.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:25 smithi077 bash[32040]: cluster 2024-09-13T00:33:23.823983+0000 mgr.y (mgr.14397) 157 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:27.524 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:27 smithi077 bash[22596]: cluster 2024-09-13T00:33:25.824534+0000 mgr.y (mgr.14397) 158 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:27.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:27 smithi077 bash[32040]: cluster 2024-09-13T00:33:25.824534+0000 mgr.y (mgr.14397) 158 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:27.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:27 smithi088 bash[24895]: cluster 2024-09-13T00:33:25.824534+0000 mgr.y (mgr.14397) 158 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:28.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:28 smithi088 bash[24895]: audit 2024-09-13T00:33:27.795907+0000 mon.c (mon.2) 103 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2024-09-13T00:33:28.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:28 smithi088 bash[24895]: audit 2024-09-13T00:33:27.796329+0000 mon.c (mon.2) 104 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-09-13T00:33:28.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:28 smithi088 bash[24895]: audit 2024-09-13T00:33:27.796650+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2024-09-13T00:33:28.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:28 smithi088 bash[24895]: audit 2024-09-13T00:33:27.796767+0000 mon.c (mon.2) 105 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-13T00:33:28.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:28 smithi088 bash[24895]: audit 2024-09-13T00:33:27.797557+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-09-13T00:33:28.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:28 smithi088 bash[24895]: audit 2024-09-13T00:33:27.798046+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-13T00:33:28.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:28 smithi088 bash[24895]: audit 2024-09-13T00:33:27.833905+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:28.661 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:28 smithi077 bash[32040]: audit 2024-09-13T00:33:27.795907+0000 mon.c (mon.2) 103 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2024-09-13T00:33:28.661 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:28 smithi077 bash[32040]: audit 2024-09-13T00:33:27.796329+0000 mon.c (mon.2) 104 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-09-13T00:33:28.661 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:28 smithi077 bash[32040]: audit 2024-09-13T00:33:27.796650+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2024-09-13T00:33:28.661 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:28 smithi077 bash[32040]: audit 2024-09-13T00:33:27.796767+0000 mon.c (mon.2) 105 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-13T00:33:28.661 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:28 smithi077 bash[32040]: audit 2024-09-13T00:33:27.797557+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-09-13T00:33:28.661 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:28 smithi077 bash[32040]: audit 2024-09-13T00:33:27.798046+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-13T00:33:28.661 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:28 smithi077 bash[32040]: audit 2024-09-13T00:33:27.833905+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:28.662 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:28 smithi077 bash[22596]: audit 2024-09-13T00:33:27.795907+0000 mon.c (mon.2) 103 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2024-09-13T00:33:28.662 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:28 smithi077 bash[22596]: audit 2024-09-13T00:33:27.796329+0000 mon.c (mon.2) 104 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-09-13T00:33:28.662 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:28 smithi077 bash[22596]: audit 2024-09-13T00:33:27.796650+0000 mon.a (mon.0) 801 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2024-09-13T00:33:28.662 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:28 smithi077 bash[22596]: audit 2024-09-13T00:33:27.796767+0000 mon.c (mon.2) 105 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-13T00:33:28.662 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:28 smithi077 bash[22596]: audit 2024-09-13T00:33:27.797557+0000 mon.a (mon.0) 802 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]: dispatch 2024-09-13T00:33:28.662 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:28 smithi077 bash[22596]: audit 2024-09-13T00:33:27.798046+0000 mon.a (mon.0) 803 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2024-09-13T00:33:28.662 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:28 smithi077 bash[22596]: audit 2024-09-13T00:33:27.833905+0000 mon.c (mon.2) 106 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:29.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:29 smithi088 bash[24895]: cluster 2024-09-13T00:33:27.825079+0000 mgr.y (mgr.14397) 159 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:29.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:29 smithi088 bash[24895]: audit 2024-09-13T00:33:28.345688+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]': finished 2024-09-13T00:33:29.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:29 smithi088 bash[24895]: audit 2024-09-13T00:33:28.345775+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-09-13T00:33:29.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:29 smithi088 bash[24895]: audit 2024-09-13T00:33:28.345839+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-13T00:33:29.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:29 smithi088 bash[24895]: cluster 2024-09-13T00:33:28.345909+0000 mon.a (mon.0) 807 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-09-13T00:33:29.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:29 smithi077 bash[22596]: cluster 2024-09-13T00:33:27.825079+0000 mgr.y (mgr.14397) 159 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:29.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:29 smithi077 bash[22596]: audit 2024-09-13T00:33:28.345688+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]': finished 2024-09-13T00:33:29.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:29 smithi077 bash[22596]: audit 2024-09-13T00:33:28.345775+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-09-13T00:33:29.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:29 smithi077 bash[22596]: audit 2024-09-13T00:33:28.345839+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-13T00:33:29.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:29 smithi077 bash[22596]: cluster 2024-09-13T00:33:28.345909+0000 mon.a (mon.0) 807 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-09-13T00:33:29.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:29 smithi077 bash[32040]: cluster 2024-09-13T00:33:27.825079+0000 mgr.y (mgr.14397) 159 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:29.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:29 smithi077 bash[32040]: audit 2024-09-13T00:33:28.345688+0000 mon.a (mon.0) 804 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]': finished 2024-09-13T00:33:29.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:29 smithi077 bash[32040]: audit 2024-09-13T00:33:28.345775+0000 mon.a (mon.0) 805 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.b", "id": [1, 2]}]': finished 2024-09-13T00:33:29.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:29 smithi077 bash[32040]: audit 2024-09-13T00:33:28.345839+0000 mon.a (mon.0) 806 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2024-09-13T00:33:29.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:29 smithi077 bash[32040]: cluster 2024-09-13T00:33:28.345909+0000 mon.a (mon.0) 807 : cluster [DBG] osdmap e64: 8 total, 8 up, 8 in 2024-09-13T00:33:30.760 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:30 smithi088 bash[24895]: cluster 2024-09-13T00:33:29.361794+0000 mon.a (mon.0) 808 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-09-13T00:33:30.774 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:30 smithi077 bash[22596]: cluster 2024-09-13T00:33:29.361794+0000 mon.a (mon.0) 808 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-09-13T00:33:30.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:30 smithi077 bash[32040]: cluster 2024-09-13T00:33:29.361794+0000 mon.a (mon.0) 808 : cluster [DBG] osdmap e65: 8 total, 8 up, 8 in 2024-09-13T00:33:31.099 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:33:30 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:33:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:33:31.774 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:31 smithi077 bash[22596]: cluster 2024-09-13T00:33:29.825685+0000 mgr.y (mgr.14397) 160 : cluster [DBG] pgmap v138: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:31.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:33:31 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:33:31] "GET /metrics HTTP/1.1" 200 240016 "" "Prometheus/2.43.0" 2024-09-13T00:33:31.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:31 smithi077 bash[32040]: cluster 2024-09-13T00:33:29.825685+0000 mgr.y (mgr.14397) 160 : cluster [DBG] pgmap v138: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:31.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:31 smithi088 bash[24895]: cluster 2024-09-13T00:33:29.825685+0000 mgr.y (mgr.14397) 160 : cluster [DBG] pgmap v138: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:32.437 INFO:teuthology.orchestra.run.smithi077.stdout:55834574974 2024-09-13T00:33:32.438 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T00:33:32.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:32 smithi077 bash[22596]: cluster 2024-09-13T00:33:32.367158+0000 mon.a (mon.0) 809 : cluster [WRN] Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-09-13T00:33:32.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:32 smithi077 bash[32040]: cluster 2024-09-13T00:33:32.367158+0000 mon.a (mon.0) 809 : cluster [WRN] Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-09-13T00:33:32.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:32 smithi088 bash[24895]: cluster 2024-09-13T00:33:32.367158+0000 mon.a (mon.0) 809 : cluster [WRN] Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2024-09-13T00:33:33.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:33 smithi077 bash[22596]: cluster 2024-09-13T00:33:31.826371+0000 mgr.y (mgr.14397) 161 : cluster [DBG] pgmap v139: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:33.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:33 smithi077 bash[32040]: cluster 2024-09-13T00:33:31.826371+0000 mgr.y (mgr.14397) 161 : cluster [DBG] pgmap v139: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:33.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:33 smithi088 bash[24895]: cluster 2024-09-13T00:33:31.826371+0000 mgr.y (mgr.14397) 161 : cluster [DBG] pgmap v139: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:34.745 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:34 smithi077 bash[22596]: cluster 2024-09-13T00:33:34.425777+0000 mon.a (mon.0) 810 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-09-13T00:33:34.745 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:34 smithi077 bash[22596]: cluster 2024-09-13T00:33:34.425827+0000 mon.a (mon.0) 811 : cluster [INF] Cluster is now healthy 2024-09-13T00:33:34.745 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:34 smithi077 bash[32040]: cluster 2024-09-13T00:33:34.425777+0000 mon.a (mon.0) 810 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-09-13T00:33:34.746 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:34 smithi077 bash[32040]: cluster 2024-09-13T00:33:34.425827+0000 mon.a (mon.0) 811 : cluster [INF] Cluster is now healthy 2024-09-13T00:33:34.790 INFO:teuthology.orchestra.run.smithi077.stdout:206158430271 2024-09-13T00:33:34.790 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T00:33:34.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:34 smithi088 bash[24895]: cluster 2024-09-13T00:33:34.425777+0000 mon.a (mon.0) 810 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs peering) 2024-09-13T00:33:34.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:34 smithi088 bash[24895]: cluster 2024-09-13T00:33:34.425827+0000 mon.a (mon.0) 811 : cluster [INF] Cluster is now healthy 2024-09-13T00:33:35.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:35 smithi088 bash[24895]: cluster 2024-09-13T00:33:33.827370+0000 mgr.y (mgr.14397) 162 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:35.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:35 smithi088 bash[24895]: audit 2024-09-13T00:33:35.239843+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:33:35.879 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:35 smithi077 bash[22596]: cluster 2024-09-13T00:33:33.827370+0000 mgr.y (mgr.14397) 162 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:35.879 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:35 smithi077 bash[22596]: audit 2024-09-13T00:33:35.239843+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:33:35.880 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:35 smithi077 bash[32040]: cluster 2024-09-13T00:33:33.827370+0000 mgr.y (mgr.14397) 162 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:35.880 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:35 smithi077 bash[32040]: audit 2024-09-13T00:33:35.239843+0000 mon.c (mon.2) 107 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:33:36.716 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:36 smithi077 bash[32040]: audit 2024-09-13T00:33:35.705897+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:33:36.717 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:36 smithi077 bash[32040]: audit 2024-09-13T00:33:35.707504+0000 mon.c (mon.2) 109 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:33:36.717 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:36 smithi077 bash[32040]: audit 2024-09-13T00:33:35.713404+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:33:36.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:36 smithi088 bash[24895]: audit 2024-09-13T00:33:35.705897+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:33:36.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:36 smithi088 bash[24895]: audit 2024-09-13T00:33:35.707504+0000 mon.c (mon.2) 109 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:33:36.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:36 smithi088 bash[24895]: audit 2024-09-13T00:33:35.713404+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:33:37.081 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:36 smithi077 bash[22596]: audit 2024-09-13T00:33:35.705897+0000 mon.c (mon.2) 108 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:33:37.081 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:36 smithi077 bash[22596]: audit 2024-09-13T00:33:35.707504+0000 mon.c (mon.2) 109 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:33:37.081 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:36 smithi077 bash[22596]: audit 2024-09-13T00:33:35.713404+0000 mon.a (mon.0) 812 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:33:37.523 INFO:teuthology.orchestra.run.smithi077.stdout:107374182503 2024-09-13T00:33:37.523 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T00:33:37.719 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:37 smithi077 bash[22596]: cluster 2024-09-13T00:33:35.828062+0000 mgr.y (mgr.14397) 163 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:37.719 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:37 smithi077 bash[32040]: cluster 2024-09-13T00:33:35.828062+0000 mgr.y (mgr.14397) 163 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:37.739 INFO:teuthology.orchestra.run.smithi077.stdout:180388626506 2024-09-13T00:33:37.740 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T00:33:37.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:37 smithi088 bash[24895]: cluster 2024-09-13T00:33:35.828062+0000 mgr.y (mgr.14397) 163 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:38.952 INFO:teuthology.orchestra.run.smithi077.stdout:34359738504 2024-09-13T00:33:38.953 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T00:33:39.137 INFO:teuthology.orchestra.run.smithi077.stdout:154618822740 2024-09-13T00:33:39.138 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T00:33:39.623 INFO:teuthology.orchestra.run.smithi077.stdout:128849018973 2024-09-13T00:33:39.624 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T00:33:39.774 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:39 smithi077 bash[32040]: cluster 2024-09-13T00:33:37.828870+0000 mgr.y (mgr.14397) 164 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:39.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:39 smithi077 bash[22596]: cluster 2024-09-13T00:33:37.828870+0000 mgr.y (mgr.14397) 164 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:39.828 INFO:teuthology.orchestra.run.smithi077.stdout:77309411443 2024-09-13T00:33:39.829 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T00:33:39.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:39 smithi088 bash[24895]: cluster 2024-09-13T00:33:37.828870+0000 mgr.y (mgr.14397) 164 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:40.298 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:41.099 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:33:40 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:33:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:33:41.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:41 smithi077 bash[22596]: cluster 2024-09-13T00:33:39.829583+0000 mgr.y (mgr.14397) 165 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:41.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:33:41 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:33:41] "GET /metrics HTTP/1.1" 200 240016 "" "Prometheus/2.43.0" 2024-09-13T00:33:41.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:41 smithi077 bash[32040]: cluster 2024-09-13T00:33:39.829583+0000 mgr.y (mgr.14397) 165 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:41.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:41 smithi088 bash[24895]: cluster 2024-09-13T00:33:39.829583+0000 mgr.y (mgr.14397) 165 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:42.864 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:42.866 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:42.866 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:43.340 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:43.341 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:43.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:43 smithi077 bash[32040]: cluster 2024-09-13T00:33:41.830001+0000 mgr.y (mgr.14397) 166 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:43.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:43 smithi077 bash[32040]: audit 2024-09-13T00:33:42.833903+0000 mon.c (mon.2) 110 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:43.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:43 smithi077 bash[22596]: cluster 2024-09-13T00:33:41.830001+0000 mgr.y (mgr.14397) 166 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:43.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:43 smithi077 bash[22596]: audit 2024-09-13T00:33:42.833903+0000 mon.c (mon.2) 110 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:43.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:43 smithi088 bash[24895]: cluster 2024-09-13T00:33:41.830001+0000 mgr.y (mgr.14397) 166 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:43.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:43 smithi088 bash[24895]: audit 2024-09-13T00:33:42.833903+0000 mon.c (mon.2) 110 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:45.265 INFO:teuthology.orchestra.run.smithi077.stdout:55834574977 2024-09-13T00:33:45.386 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:45.386 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:33:45.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:45 smithi077 bash[22596]: cluster 2024-09-13T00:33:43.831032+0000 mgr.y (mgr.14397) 167 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:45.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:45 smithi077 bash[22596]: audit 2024-09-13T00:33:45.271962+0000 mon.a (mon.0) 813 : audit [DBG] from='client.? 172.21.15.77:0/3189560822' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T00:33:45.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:45 smithi077 bash[32040]: cluster 2024-09-13T00:33:43.831032+0000 mgr.y (mgr.14397) 167 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:45.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:45 smithi077 bash[32040]: audit 2024-09-13T00:33:45.271962+0000 mon.a (mon.0) 813 : audit [DBG] from='client.? 172.21.15.77:0/3189560822' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T00:33:45.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:45 smithi088 bash[24895]: cluster 2024-09-13T00:33:43.831032+0000 mgr.y (mgr.14397) 167 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:45.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:45 smithi088 bash[24895]: audit 2024-09-13T00:33:45.271962+0000 mon.a (mon.0) 813 : audit [DBG] from='client.? 172.21.15.77:0/3189560822' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T00:33:47.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:47 smithi077 bash[22596]: cluster 2024-09-13T00:33:45.831595+0000 mgr.y (mgr.14397) 168 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:47.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:47 smithi077 bash[32040]: cluster 2024-09-13T00:33:45.831595+0000 mgr.y (mgr.14397) 168 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:47.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:47 smithi088 bash[24895]: cluster 2024-09-13T00:33:45.831595+0000 mgr.y (mgr.14397) 168 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:48.535 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574974 got 55834574977 for osd.1 2024-09-13T00:33:48.535 DEBUG:teuthology.parallel:result is None 2024-09-13T00:33:49.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:49 smithi077 bash[22596]: cluster 2024-09-13T00:33:47.832157+0000 mgr.y (mgr.14397) 169 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:49.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:49 smithi077 bash[32040]: cluster 2024-09-13T00:33:47.832157+0000 mgr.y (mgr.14397) 169 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:49.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:49 smithi088 bash[24895]: cluster 2024-09-13T00:33:47.832157+0000 mgr.y (mgr.14397) 169 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:51.099 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:33:50 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:33:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:33:51.810 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:33:51 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:33:51] "GET /metrics HTTP/1.1" 200 240053 "" "Prometheus/2.43.0" 2024-09-13T00:33:51.810 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:51 smithi077 bash[32040]: cluster 2024-09-13T00:33:49.833225+0000 mgr.y (mgr.14397) 170 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:51.811 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:51 smithi077 bash[22596]: cluster 2024-09-13T00:33:49.833225+0000 mgr.y (mgr.14397) 170 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:51.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:51 smithi088 bash[24895]: cluster 2024-09-13T00:33:49.833225+0000 mgr.y (mgr.14397) 170 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:53.762 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:53 smithi077 bash[22596]: cluster 2024-09-13T00:33:51.833841+0000 mgr.y (mgr.14397) 171 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:53.762 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:53 smithi077 bash[32040]: cluster 2024-09-13T00:33:51.833841+0000 mgr.y (mgr.14397) 171 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:53.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:53 smithi088 bash[24895]: cluster 2024-09-13T00:33:51.833841+0000 mgr.y (mgr.14397) 171 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:55.135 INFO:teuthology.orchestra.run.smithi077.stdout:180388626510 2024-09-13T00:33:55.811 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:55 smithi077 bash[22596]: cluster 2024-09-13T00:33:53.834577+0000 mgr.y (mgr.14397) 172 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:55.811 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:55 smithi077 bash[22596]: audit 2024-09-13T00:33:55.141381+0000 mon.a (mon.0) 814 : audit [DBG] from='client.? 172.21.15.77:0/3163023251' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T00:33:55.812 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:55 smithi077 bash[32040]: cluster 2024-09-13T00:33:53.834577+0000 mgr.y (mgr.14397) 172 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:55.812 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:55 smithi077 bash[32040]: audit 2024-09-13T00:33:55.141381+0000 mon.a (mon.0) 814 : audit [DBG] from='client.? 172.21.15.77:0/3163023251' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T00:33:55.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:55 smithi088 bash[24895]: cluster 2024-09-13T00:33:53.834577+0000 mgr.y (mgr.14397) 172 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:55.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:55 smithi088 bash[24895]: audit 2024-09-13T00:33:55.141381+0000 mon.a (mon.0) 814 : audit [DBG] from='client.? 172.21.15.77:0/3163023251' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T00:33:57.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:57 smithi077 bash[22596]: cluster 2024-09-13T00:33:55.835114+0000 mgr.y (mgr.14397) 173 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:57.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:57 smithi077 bash[32040]: cluster 2024-09-13T00:33:55.835114+0000 mgr.y (mgr.14397) 173 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:57.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:57 smithi088 bash[24895]: cluster 2024-09-13T00:33:55.835114+0000 mgr.y (mgr.14397) 173 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:58.598 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:58 smithi077 bash[22596]: audit 2024-09-13T00:33:57.834634+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:58.599 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:58 smithi077 bash[22596]: cluster 2024-09-13T00:33:57.835736+0000 mgr.y (mgr.14397) 174 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:58.599 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:58 smithi077 bash[32040]: audit 2024-09-13T00:33:57.834634+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:58.599 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:58 smithi077 bash[32040]: cluster 2024-09-13T00:33:57.835736+0000 mgr.y (mgr.14397) 174 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:58.600 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626506 got 180388626510 for osd.6 2024-09-13T00:33:58.600 DEBUG:teuthology.parallel:result is None 2024-09-13T00:33:58.613 INFO:teuthology.orchestra.run.smithi077.stdout:34359738508 2024-09-13T00:33:58.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:58 smithi088 bash[24895]: audit 2024-09-13T00:33:57.834634+0000 mon.c (mon.2) 111 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:33:58.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:58 smithi088 bash[24895]: cluster 2024-09-13T00:33:57.835736+0000 mgr.y (mgr.14397) 174 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:33:59.240 INFO:teuthology.orchestra.run.smithi077.stdout:206158430276 2024-09-13T00:33:59.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:59 smithi088 bash[24895]: audit 2024-09-13T00:33:58.620165+0000 mon.a (mon.0) 815 : audit [DBG] from='client.? 172.21.15.77:0/3385105778' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T00:33:59.849 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:33:59 smithi088 bash[24895]: audit 2024-09-13T00:33:59.246838+0000 mon.a (mon.0) 816 : audit [DBG] from='client.? 172.21.15.77:0/4245859246' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T00:33:59.980 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:59 smithi077 bash[22596]: audit 2024-09-13T00:33:58.620165+0000 mon.a (mon.0) 815 : audit [DBG] from='client.? 172.21.15.77:0/3385105778' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T00:33:59.980 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:33:59 smithi077 bash[22596]: audit 2024-09-13T00:33:59.246838+0000 mon.a (mon.0) 816 : audit [DBG] from='client.? 172.21.15.77:0/4245859246' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T00:33:59.981 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:59 smithi077 bash[32040]: audit 2024-09-13T00:33:58.620165+0000 mon.a (mon.0) 815 : audit [DBG] from='client.? 172.21.15.77:0/3385105778' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T00:33:59.981 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:33:59 smithi077 bash[32040]: audit 2024-09-13T00:33:59.246838+0000 mon.a (mon.0) 816 : audit [DBG] from='client.? 172.21.15.77:0/4245859246' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T00:34:00.371 INFO:teuthology.orchestra.run.smithi077.stdout:154618822744 2024-09-13T00:34:00.694 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:00 smithi077 bash[22596]: cluster 2024-09-13T00:33:59.836800+0000 mgr.y (mgr.14397) 175 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:00.694 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:00 smithi077 bash[22596]: audit 2024-09-13T00:34:00.378314+0000 mon.a (mon.0) 817 : audit [DBG] from='client.? 172.21.15.77:0/3437146083' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T00:34:00.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:00 smithi077 bash[32040]: cluster 2024-09-13T00:33:59.836800+0000 mgr.y (mgr.14397) 175 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:00.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:00 smithi077 bash[32040]: audit 2024-09-13T00:34:00.378314+0000 mon.a (mon.0) 817 : audit [DBG] from='client.? 172.21.15.77:0/3437146083' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T00:34:00.849 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:34:00 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:34:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:34:00.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:00 smithi088 bash[24895]: cluster 2024-09-13T00:33:59.836800+0000 mgr.y (mgr.14397) 175 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:00.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:00 smithi088 bash[24895]: audit 2024-09-13T00:34:00.378314+0000 mon.a (mon.0) 817 : audit [DBG] from='client.? 172.21.15.77:0/3437146083' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T00:34:00.962 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738504 got 34359738508 for osd.0 2024-09-13T00:34:00.963 DEBUG:teuthology.parallel:result is None 2024-09-13T00:34:01.638 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430271 got 206158430276 for osd.7 2024-09-13T00:34:01.639 DEBUG:teuthology.parallel:result is None 2024-09-13T00:34:01.743 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:34:01 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:34:01] "GET /metrics HTTP/1.1" 200 240038 "" "Prometheus/2.43.0" 2024-09-13T00:34:02.531 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822740 got 154618822744 for osd.5 2024-09-13T00:34:02.531 DEBUG:teuthology.parallel:result is None 2024-09-13T00:34:02.938 INFO:teuthology.orchestra.run.smithi077.stdout:128849018978 2024-09-13T00:34:02.962 INFO:teuthology.orchestra.run.smithi077.stdout:77309411448 2024-09-13T00:34:02.996 INFO:teuthology.orchestra.run.smithi077.stdout:107374182508 2024-09-13T00:34:03.267 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:02 smithi077 bash[22596]: cluster 2024-09-13T00:34:01.837426+0000 mgr.y (mgr.14397) 176 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:03.267 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:02 smithi077 bash[32040]: cluster 2024-09-13T00:34:01.837426+0000 mgr.y (mgr.14397) 176 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:03.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:02 smithi088 bash[24895]: cluster 2024-09-13T00:34:01.837426+0000 mgr.y (mgr.14397) 176 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:04.201 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:03 smithi077 bash[22596]: audit 2024-09-13T00:34:02.945252+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.77:0/3671531264' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T00:34:04.202 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:03 smithi077 bash[22596]: audit 2024-09-13T00:34:02.968645+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.77:0/406090075' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T00:34:04.202 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:03 smithi077 bash[22596]: audit 2024-09-13T00:34:03.002829+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.77:0/1196548490' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T00:34:04.202 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:03 smithi077 bash[32040]: audit 2024-09-13T00:34:02.945252+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.77:0/3671531264' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T00:34:04.202 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:03 smithi077 bash[32040]: audit 2024-09-13T00:34:02.968645+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.77:0/406090075' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T00:34:04.202 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:03 smithi077 bash[32040]: audit 2024-09-13T00:34:03.002829+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.77:0/1196548490' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T00:34:04.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:03 smithi088 bash[24895]: audit 2024-09-13T00:34:02.945252+0000 mon.c (mon.2) 112 : audit [DBG] from='client.? 172.21.15.77:0/3671531264' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T00:34:04.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:03 smithi088 bash[24895]: audit 2024-09-13T00:34:02.968645+0000 mon.c (mon.2) 113 : audit [DBG] from='client.? 172.21.15.77:0/406090075' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T00:34:04.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:03 smithi088 bash[24895]: audit 2024-09-13T00:34:03.002829+0000 mon.c (mon.2) 114 : audit [DBG] from='client.? 172.21.15.77:0/1196548490' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T00:34:04.886 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182503 got 107374182508 for osd.3 2024-09-13T00:34:04.886 DEBUG:teuthology.parallel:result is None 2024-09-13T00:34:04.925 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411443 got 77309411448 for osd.2 2024-09-13T00:34:04.925 DEBUG:teuthology.parallel:result is None 2024-09-13T00:34:05.011 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018973 got 128849018978 for osd.4 2024-09-13T00:34:05.012 DEBUG:teuthology.parallel:result is None 2024-09-13T00:34:05.012 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T00:34:05.012 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T00:34:05.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:04 smithi077 bash[22596]: cluster 2024-09-13T00:34:03.838414+0000 mgr.y (mgr.14397) 177 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:05.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:04 smithi077 bash[32040]: cluster 2024-09-13T00:34:03.838414+0000 mgr.y (mgr.14397) 177 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:05.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:04 smithi088 bash[24895]: cluster 2024-09-13T00:34:03.838414+0000 mgr.y (mgr.14397) 177 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:07.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:06 smithi077 bash[22596]: cluster 2024-09-13T00:34:05.839063+0000 mgr.y (mgr.14397) 178 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:07.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:06 smithi077 bash[32040]: cluster 2024-09-13T00:34:05.839063+0000 mgr.y (mgr.14397) 178 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:07.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:06 smithi088 bash[24895]: cluster 2024-09-13T00:34:05.839063+0000 mgr.y (mgr.14397) 178 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:09.274 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:08 smithi077 bash[22596]: cluster 2024-09-13T00:34:07.839751+0000 mgr.y (mgr.14397) 179 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:09.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:08 smithi077 bash[32040]: cluster 2024-09-13T00:34:07.839751+0000 mgr.y (mgr.14397) 179 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:09.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:08 smithi088 bash[24895]: cluster 2024-09-13T00:34:07.839751+0000 mgr.y (mgr.14397) 179 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:09.791 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:34:11.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:10 smithi088 bash[24895]: cluster 2024-09-13T00:34:09.840540+0000 mgr.y (mgr.14397) 180 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:11.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:34:10 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:34:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:34:11.201 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:10 smithi077 bash[32040]: cluster 2024-09-13T00:34:09.840540+0000 mgr.y (mgr.14397) 180 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:11.202 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:10 smithi077 bash[22596]: cluster 2024-09-13T00:34:09.840540+0000 mgr.y (mgr.14397) 180 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:11.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:34:11 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:34:11] "GET /metrics HTTP/1.1" 200 240038 "" "Prometheus/2.43.0" 2024-09-13T00:34:13.182 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:12 smithi077 bash[22596]: cluster 2024-09-13T00:34:11.841131+0000 mgr.y (mgr.14397) 181 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:13.182 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:12 smithi077 bash[22596]: audit 2024-09-13T00:34:12.834503+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:13.182 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:12 smithi077 bash[32040]: cluster 2024-09-13T00:34:11.841131+0000 mgr.y (mgr.14397) 181 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:13.183 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:12 smithi077 bash[32040]: audit 2024-09-13T00:34:12.834503+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:13.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:12 smithi088 bash[24895]: cluster 2024-09-13T00:34:11.841131+0000 mgr.y (mgr.14397) 181 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:13.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:12 smithi088 bash[24895]: audit 2024-09-13T00:34:12.834503+0000 mon.c (mon.2) 115 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:13.483 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:34:13.484 INFO:teuthology.orchestra.run.smithi077.stderr:dumped all 2024-09-13T00:34:14.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:13 smithi077 bash[22596]: audit 2024-09-13T00:34:13.479039+0000 mgr.y (mgr.14397) 182 : audit [DBG] from='client.24620 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:34:14.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:13 smithi077 bash[32040]: audit 2024-09-13T00:34:13.479039+0000 mgr.y (mgr.14397) 182 : audit [DBG] from='client.24620 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:34:14.278 INFO:teuthology.orchestra.run.smithi077.stdout:{"pg_ready":true,"pg_map":{"version":159,"stamp":"2024-09-13T00:34:11.840821+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592725,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":173,"num_read_kb":280,"num_write":160,"num_write_kb":1643,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127,"ondisk_log_size":127,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385880,"kb_used_data":7016,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575336,"statfs":{"total":767960285184,"available":765517144064,"internally_reserved":0,"allocated":7184384,"data_stored":4129313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004789"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.355068+0000","last_change":"2024-09-13T00:32:23.860928+0000","last_active":"2024-09-13T00:33:28.355068+0000","last_peered":"2024-09-13T00:33:28.355068+0000","last_clean":"2024-09-13T00:33:28.355068+0000","last_became_active":"2024-09-13T00:32:23.860092+0000","last_became_peered":"2024-09-13T00:32:23.860092+0000","last_unstale":"2024-09-13T00:33:28.355068+0000","last_undegraded":"2024-09-13T00:33:28.355068+0000","last_fullsized":"2024-09-13T00:33:28.355068+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:26:04.784182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"62'2","reported_seq":18,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.790845+0000","last_change":"2024-09-13T00:32:23.110415+0000","last_active":"2024-09-13T00:33:28.790845+0000","last_peered":"2024-09-13T00:33:28.790845+0000","last_clean":"2024-09-13T00:33:28.790845+0000","last_became_active":"2024-09-13T00:32:23.109400+0000","last_became_peered":"2024-09-13T00:32:23.109400+0000","last_unstale":"2024-09-13T00:33:28.790845+0000","last_undegraded":"2024-09-13T00:33:28.790845+0000","last_fullsized":"2024-09-13T00:33:28.790845+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:04:15.530283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352278+0000","last_change":"2024-09-13T00:32:23.861242+0000","last_active":"2024-09-13T00:33:28.352278+0000","last_peered":"2024-09-13T00:33:28.352278+0000","last_clean":"2024-09-13T00:33:28.352278+0000","last_became_active":"2024-09-13T00:32:23.860422+0000","last_became_peered":"2024-09-13T00:32:23.860422+0000","last_unstale":"2024-09-13T00:33:28.352278+0000","last_undegraded":"2024-09-13T00:33:28.352278+0000","last_fullsized":"2024-09-13T00:33:28.352278+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:34:40.070133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.345573+0000","last_change":"2024-09-13T00:32:23.128078+0000","last_active":"2024-09-13T00:33:28.345573+0000","last_peered":"2024-09-13T00:33:28.345573+0000","last_clean":"2024-09-13T00:33:28.345573+0000","last_became_active":"2024-09-13T00:32:23.127612+0000","last_became_peered":"2024-09-13T00:32:23.127612+0000","last_unstale":"2024-09-13T00:33:28.345573+0000","last_undegraded":"2024-09-13T00:33:28.345573+0000","last_fullsized":"2024-09-13T00:33:28.345573+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:36:20.193555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.347496+0000","last_change":"2024-09-13T00:32:23.863860+0000","last_active":"2024-09-13T00:33:28.347496+0000","last_peered":"2024-09-13T00:33:28.347496+0000","last_clean":"2024-09-13T00:33:28.347496+0000","last_became_active":"2024-09-13T00:32:23.861277+0000","last_became_peered":"2024-09-13T00:32:23.861277+0000","last_unstale":"2024-09-13T00:33:28.347496+0000","last_undegraded":"2024-09-13T00:33:28.347496+0000","last_fullsized":"2024-09-13T00:33:28.347496+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:32:46.629677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.790534+0000","last_change":"2024-09-13T00:32:23.110220+0000","last_active":"2024-09-13T00:33:28.790534+0000","last_peered":"2024-09-13T00:33:28.790534+0000","last_clean":"2024-09-13T00:33:28.790534+0000","last_became_active":"2024-09-13T00:32:23.109022+0000","last_became_peered":"2024-09-13T00:32:23.109022+0000","last_unstale":"2024-09-13T00:33:28.790534+0000","last_undegraded":"2024-09-13T00:33:28.790534+0000","last_fullsized":"2024-09-13T00:33:28.790534+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:23:26.497941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.355372+0000","last_change":"2024-09-13T00:32:23.860610+0000","last_active":"2024-09-13T00:33:28.355372+0000","last_peered":"2024-09-13T00:33:28.355372+0000","last_clean":"2024-09-13T00:33:28.355372+0000","last_became_active":"2024-09-13T00:32:23.859814+0000","last_became_peered":"2024-09-13T00:32:23.859814+0000","last_unstale":"2024-09-13T00:33:28.355372+0000","last_undegraded":"2024-09-13T00:33:28.355372+0000","last_fullsized":"2024-09-13T00:33:28.355372+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:06:43.248402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"62'8","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.353028+0000","last_change":"2024-09-13T00:32:23.122582+0000","last_active":"2024-09-13T00:33:28.353028+0000","last_peered":"2024-09-13T00:33:28.353028+0000","last_clean":"2024-09-13T00:33:28.353028+0000","last_became_active":"2024-09-13T00:32:23.121812+0000","last_became_peered":"2024-09-13T00:32:23.121812+0000","last_unstale":"2024-09-13T00:33:28.353028+0000","last_undegraded":"2024-09-13T00:33:28.353028+0000","last_fullsized":"2024-09-13T00:33:28.353028+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:29:49.937607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"62'1","reported_seq":16,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.791359+0000","last_change":"2024-09-13T00:32:23.121516+0000","last_active":"2024-09-13T00:33:28.791359+0000","last_peered":"2024-09-13T00:33:28.791359+0000","last_clean":"2024-09-13T00:33:28.791359+0000","last_became_active":"2024-09-13T00:32:23.120641+0000","last_became_peered":"2024-09-13T00:32:23.120641+0000","last_unstale":"2024-09-13T00:33:28.791359+0000","last_undegraded":"2024-09-13T00:33:28.791359+0000","last_fullsized":"2024-09-13T00:33:28.791359+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:35:52.434687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.350681+0000","last_change":"2024-09-13T00:32:23.861007+0000","last_active":"2024-09-13T00:33:28.350681+0000","last_peered":"2024-09-13T00:33:28.350681+0000","last_clean":"2024-09-13T00:33:28.350681+0000","last_became_active":"2024-09-13T00:32:23.860213+0000","last_became_peered":"2024-09-13T00:32:23.860213+0000","last_unstale":"2024-09-13T00:33:28.350681+0000","last_undegraded":"2024-09-13T00:33:28.350681+0000","last_fullsized":"2024-09-13T00:33:28.350681+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:58:20.866610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"62'2","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.791012+0000","last_change":"2024-09-13T00:32:23.109098+0000","last_active":"2024-09-13T00:33:28.791012+0000","last_peered":"2024-09-13T00:33:28.791012+0000","last_clean":"2024-09-13T00:33:28.791012+0000","last_became_active":"2024-09-13T00:32:23.108209+0000","last_became_peered":"2024-09-13T00:32:23.108209+0000","last_unstale":"2024-09-13T00:33:28.791012+0000","last_undegraded":"2024-09-13T00:33:28.791012+0000","last_fullsized":"2024-09-13T00:33:28.791012+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:55:33.433874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":23,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T00:33:29.788338+0000","last_change":"2024-09-13T00:33:29.788338+0000","last_active":"2024-09-13T00:33:29.788338+0000","last_peered":"2024-09-13T00:33:29.788338+0000","last_clean":"2024-09-13T00:33:29.788338+0000","last_became_active":"2024-09-13T00:33:29.787460+0000","last_became_peered":"2024-09-13T00:33:29.787460+0000","last_unstale":"2024-09-13T00:33:29.788338+0000","last_undegraded":"2024-09-13T00:33:29.788338+0000","last_fullsized":"2024-09-13T00:33:29.788338+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:17:54.554758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"62'1","reported_seq":16,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.791312+0000","last_change":"2024-09-13T00:32:23.122340+0000","last_active":"2024-09-13T00:33:28.791312+0000","last_peered":"2024-09-13T00:33:28.791312+0000","last_clean":"2024-09-13T00:33:28.791312+0000","last_became_active":"2024-09-13T00:32:23.121795+0000","last_became_peered":"2024-09-13T00:32:23.121795+0000","last_unstale":"2024-09-13T00:33:28.791312+0000","last_undegraded":"2024-09-13T00:33:28.791312+0000","last_fullsized":"2024-09-13T00:33:28.791312+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:18:28.661306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.350314+0000","last_change":"2024-09-13T00:32:23.861336+0000","last_active":"2024-09-13T00:33:28.350314+0000","last_peered":"2024-09-13T00:33:28.350314+0000","last_clean":"2024-09-13T00:33:28.350314+0000","last_became_active":"2024-09-13T00:32:23.860083+0000","last_became_peered":"2024-09-13T00:32:23.860083+0000","last_unstale":"2024-09-13T00:33:28.350314+0000","last_undegraded":"2024-09-13T00:33:28.350314+0000","last_fullsized":"2024-09-13T00:33:28.350314+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:18:50.519864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352085+0000","last_change":"2024-09-13T00:32:23.118671+0000","last_active":"2024-09-13T00:33:28.352085+0000","last_peered":"2024-09-13T00:33:28.352085+0000","last_clean":"2024-09-13T00:33:28.352085+0000","last_became_active":"2024-09-13T00:32:23.113524+0000","last_became_peered":"2024-09-13T00:32:23.113524+0000","last_unstale":"2024-09-13T00:33:28.352085+0000","last_undegraded":"2024-09-13T00:33:28.352085+0000","last_fullsized":"2024-09-13T00:33:28.352085+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:08:35.149880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":23,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T00:33:29.364966+0000","last_change":"2024-09-13T00:33:29.364966+0000","last_active":"2024-09-13T00:33:29.364966+0000","last_peered":"2024-09-13T00:33:29.364966+0000","last_clean":"2024-09-13T00:33:29.364966+0000","last_became_active":"2024-09-13T00:33:29.364191+0000","last_became_peered":"2024-09-13T00:33:29.364191+0000","last_unstale":"2024-09-13T00:33:29.364966+0000","last_undegraded":"2024-09-13T00:33:29.364966+0000","last_fullsized":"2024-09-13T00:33:29.364966+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:45:42.373334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"63'3","reported_seq":18,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.354637+0000","last_change":"2024-09-13T00:32:23.118698+0000","last_active":"2024-09-13T00:33:28.354637+0000","last_peered":"2024-09-13T00:33:28.354637+0000","last_clean":"2024-09-13T00:33:28.354637+0000","last_became_active":"2024-09-13T00:32:23.118060+0000","last_became_peered":"2024-09-13T00:32:23.118060+0000","last_unstale":"2024-09-13T00:33:28.354637+0000","last_undegraded":"2024-09-13T00:33:28.354637+0000","last_fullsized":"2024-09-13T00:33:28.354637+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:32:06.783542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.345276+0000","last_change":"2024-09-13T00:32:23.168009+0000","last_active":"2024-09-13T00:33:28.345276+0000","last_peered":"2024-09-13T00:33:28.345276+0000","last_clean":"2024-09-13T00:33:28.345276+0000","last_became_active":"2024-09-13T00:32:23.167303+0000","last_became_peered":"2024-09-13T00:32:23.167303+0000","last_unstale":"2024-09-13T00:33:28.345276+0000","last_undegraded":"2024-09-13T00:33:28.345276+0000","last_fullsized":"2024-09-13T00:33:28.345276+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:45:27.608084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.350684+0000","last_change":"2024-09-13T00:32:23.096317+0000","last_active":"2024-09-13T00:33:28.350684+0000","last_peered":"2024-09-13T00:33:28.350684+0000","last_clean":"2024-09-13T00:33:28.350684+0000","last_became_active":"2024-09-13T00:32:23.095610+0000","last_became_peered":"2024-09-13T00:32:23.095610+0000","last_unstale":"2024-09-13T00:33:28.350684+0000","last_undegraded":"2024-09-13T00:33:28.350684+0000","last_fullsized":"2024-09-13T00:33:28.350684+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:32:37.463877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.346418+0000","last_change":"2024-09-13T00:32:23.160451+0000","last_active":"2024-09-13T00:33:28.346418+0000","last_peered":"2024-09-13T00:33:28.346418+0000","last_clean":"2024-09-13T00:33:28.346418+0000","last_became_active":"2024-09-13T00:32:23.158923+0000","last_became_peered":"2024-09-13T00:32:23.158923+0000","last_unstale":"2024-09-13T00:33:28.346418+0000","last_undegraded":"2024-09-13T00:33:28.346418+0000","last_fullsized":"2024-09-13T00:33:28.346418+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:07:10.841651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.345665+0000","last_change":"2024-09-13T00:32:23.103760+0000","last_active":"2024-09-13T00:33:28.345665+0000","last_peered":"2024-09-13T00:33:28.345665+0000","last_clean":"2024-09-13T00:33:28.345665+0000","last_became_active":"2024-09-13T00:32:23.102653+0000","last_became_peered":"2024-09-13T00:32:23.102653+0000","last_unstale":"2024-09-13T00:33:28.345665+0000","last_undegraded":"2024-09-13T00:33:28.345665+0000","last_fullsized":"2024-09-13T00:33:28.345665+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:10:24.285084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.346342+0000","last_change":"2024-09-13T00:32:23.159864+0000","last_active":"2024-09-13T00:33:28.346342+0000","last_peered":"2024-09-13T00:33:28.346342+0000","last_clean":"2024-09-13T00:33:28.346342+0000","last_became_active":"2024-09-13T00:32:23.158975+0000","last_became_peered":"2024-09-13T00:32:23.158975+0000","last_unstale":"2024-09-13T00:33:28.346342+0000","last_undegraded":"2024-09-13T00:33:28.346342+0000","last_fullsized":"2024-09-13T00:33:28.346342+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:42:46.526231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352514+0000","last_change":"2024-09-13T00:32:23.133548+0000","last_active":"2024-09-13T00:33:28.352514+0000","last_peered":"2024-09-13T00:33:28.352514+0000","last_clean":"2024-09-13T00:33:28.352514+0000","last_became_active":"2024-09-13T00:32:23.132627+0000","last_became_peered":"2024-09-13T00:32:23.132627+0000","last_unstale":"2024-09-13T00:33:28.352514+0000","last_undegraded":"2024-09-13T00:33:28.352514+0000","last_fullsized":"2024-09-13T00:33:28.352514+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:46:28.698050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.353047+0000","last_change":"2024-09-13T00:32:23.173231+0000","last_active":"2024-09-13T00:33:28.353047+0000","last_peered":"2024-09-13T00:33:28.353047+0000","last_clean":"2024-09-13T00:33:28.353047+0000","last_became_active":"2024-09-13T00:32:23.172678+0000","last_became_peered":"2024-09-13T00:32:23.172678+0000","last_unstale":"2024-09-13T00:33:28.353047+0000","last_undegraded":"2024-09-13T00:33:28.353047+0000","last_fullsized":"2024-09-13T00:33:28.353047+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:07:25.182130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"51'79","reported_seq":56,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.790967+0000","last_change":"2024-09-13T00:28:31.831547+0000","last_active":"2024-09-13T00:33:28.790967+0000","last_peered":"2024-09-13T00:33:28.790967+0000","last_clean":"2024-09-13T00:33:28.790967+0000","last_became_active":"2024-09-13T00:28:31.819822+0000","last_became_peered":"2024-09-13T00:28:31.819822+0000","last_unstale":"2024-09-13T00:33:28.790967+0000","last_undegraded":"2024-09-13T00:33:28.790967+0000","last_fullsized":"2024-09-13T00:33:28.790967+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:24:16.220413+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:24:16.220413+0000","last_clean_scrub_stamp":"2024-09-13T00:24:16.220413+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:19:22.270602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"55'1","reported_seq":27,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351748+0000","last_change":"2024-09-13T00:30:22.058593+0000","last_active":"2024-09-13T00:33:28.351748+0000","last_peered":"2024-09-13T00:33:28.351748+0000","last_clean":"2024-09-13T00:33:28.351748+0000","last_became_active":"2024-09-13T00:30:11.000634+0000","last_became_peered":"2024-09-13T00:30:11.000634+0000","last_unstale":"2024-09-13T00:33:28.351748+0000","last_undegraded":"2024-09-13T00:33:28.351748+0000","last_fullsized":"2024-09-13T00:33:28.351748+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:18:10.848845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096918099999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.353016+0000","last_change":"2024-09-13T00:32:23.118706+0000","last_active":"2024-09-13T00:33:28.353016+0000","last_peered":"2024-09-13T00:33:28.353016+0000","last_clean":"2024-09-13T00:33:28.353016+0000","last_became_active":"2024-09-13T00:32:23.117537+0000","last_became_peered":"2024-09-13T00:32:23.117537+0000","last_unstale":"2024-09-13T00:33:28.353016+0000","last_undegraded":"2024-09-13T00:33:28.353016+0000","last_fullsized":"2024-09-13T00:33:28.353016+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:40:21.484822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351986+0000","last_change":"2024-09-13T00:32:23.863503+0000","last_active":"2024-09-13T00:33:28.351986+0000","last_peered":"2024-09-13T00:33:28.351986+0000","last_clean":"2024-09-13T00:33:28.351986+0000","last_became_active":"2024-09-13T00:32:23.862635+0000","last_became_peered":"2024-09-13T00:32:23.862635+0000","last_unstale":"2024-09-13T00:33:28.351986+0000","last_undegraded":"2024-09-13T00:33:28.351986+0000","last_fullsized":"2024-09-13T00:33:28.351986+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:19:46.365911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.790736+0000","last_change":"2024-09-13T00:30:22.179791+0000","last_active":"2024-09-13T00:33:28.790736+0000","last_peered":"2024-09-13T00:33:28.790736+0000","last_clean":"2024-09-13T00:33:28.790736+0000","last_became_active":"2024-09-13T00:30:11.064343+0000","last_became_peered":"2024-09-13T00:30:11.064343+0000","last_unstale":"2024-09-13T00:33:28.790736+0000","last_undegraded":"2024-09-13T00:33:28.790736+0000","last_fullsized":"2024-09-13T00:33:28.790736+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:40:42.486916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062469399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.345404+0000","last_change":"2024-09-13T00:32:23.122555+0000","last_active":"2024-09-13T00:33:28.345404+0000","last_peered":"2024-09-13T00:33:28.345404+0000","last_clean":"2024-09-13T00:33:28.345404+0000","last_became_active":"2024-09-13T00:32:23.118952+0000","last_became_peered":"2024-09-13T00:32:23.118952+0000","last_unstale":"2024-09-13T00:33:28.345404+0000","last_undegraded":"2024-09-13T00:33:28.345404+0000","last_fullsized":"2024-09-13T00:33:28.345404+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:16:12.925852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.347155+0000","last_change":"2024-09-13T00:32:23.162688+0000","last_active":"2024-09-13T00:33:28.347155+0000","last_peered":"2024-09-13T00:33:28.347155+0000","last_clean":"2024-09-13T00:33:28.347155+0000","last_became_active":"2024-09-13T00:32:23.161593+0000","last_became_peered":"2024-09-13T00:32:23.161593+0000","last_unstale":"2024-09-13T00:33:28.347155+0000","last_undegraded":"2024-09-13T00:33:28.347155+0000","last_fullsized":"2024-09-13T00:33:28.347155+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:39:41.367544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.346694+0000","last_change":"2024-09-13T00:30:22.048961+0000","last_active":"2024-09-13T00:33:28.346694+0000","last_peered":"2024-09-13T00:33:28.346694+0000","last_clean":"2024-09-13T00:33:28.346694+0000","last_became_active":"2024-09-13T00:30:11.063178+0000","last_became_peered":"2024-09-13T00:30:11.063178+0000","last_unstale":"2024-09-13T00:33:28.346694+0000","last_undegraded":"2024-09-13T00:33:28.346694+0000","last_fullsized":"2024-09-13T00:33:28.346694+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:15:51.153544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067555600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351826+0000","last_change":"2024-09-13T00:32:23.108212+0000","last_active":"2024-09-13T00:33:28.351826+0000","last_peered":"2024-09-13T00:33:28.351826+0000","last_clean":"2024-09-13T00:33:28.351826+0000","last_became_active":"2024-09-13T00:32:23.103191+0000","last_became_peered":"2024-09-13T00:32:23.103191+0000","last_unstale":"2024-09-13T00:33:28.351826+0000","last_undegraded":"2024-09-13T00:33:28.351826+0000","last_fullsized":"2024-09-13T00:33:28.351826+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:50:25.094778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.353256+0000","last_change":"2024-09-13T00:32:23.160087+0000","last_active":"2024-09-13T00:33:28.353256+0000","last_peered":"2024-09-13T00:33:28.353256+0000","last_clean":"2024-09-13T00:33:28.353256+0000","last_became_active":"2024-09-13T00:32:23.159109+0000","last_became_peered":"2024-09-13T00:32:23.159109+0000","last_unstale":"2024-09-13T00:33:28.353256+0000","last_undegraded":"2024-09-13T00:33:28.353256+0000","last_fullsized":"2024-09-13T00:33:28.353256+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:13:08.870421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"57'2","reported_seq":28,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352114+0000","last_change":"2024-09-13T00:30:22.059692+0000","last_active":"2024-09-13T00:33:28.352114+0000","last_peered":"2024-09-13T00:33:28.352114+0000","last_clean":"2024-09-13T00:33:28.352114+0000","last_became_active":"2024-09-13T00:30:11.001662+0000","last_became_peered":"2024-09-13T00:30:11.001662+0000","last_unstale":"2024-09-13T00:33:28.352114+0000","last_undegraded":"2024-09-13T00:33:28.352114+0000","last_fullsized":"2024-09-13T00:33:28.352114+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:11:06.803316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002059707,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.347218+0000","last_change":"2024-09-13T00:32:23.118129+0000","last_active":"2024-09-13T00:33:28.347218+0000","last_peered":"2024-09-13T00:33:28.347218+0000","last_clean":"2024-09-13T00:33:28.347218+0000","last_became_active":"2024-09-13T00:32:23.116978+0000","last_became_peered":"2024-09-13T00:32:23.116978+0000","last_unstale":"2024-09-13T00:33:28.347218+0000","last_undegraded":"2024-09-13T00:33:28.347218+0000","last_fullsized":"2024-09-13T00:33:28.347218+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:08:03.526409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.353885+0000","last_change":"2024-09-13T00:32:23.172972+0000","last_active":"2024-09-13T00:33:28.353885+0000","last_peered":"2024-09-13T00:33:28.353885+0000","last_clean":"2024-09-13T00:33:28.353885+0000","last_became_active":"2024-09-13T00:32:23.172626+0000","last_became_peered":"2024-09-13T00:32:23.172626+0000","last_unstale":"2024-09-13T00:33:28.353885+0000","last_undegraded":"2024-09-13T00:33:28.353885+0000","last_fullsized":"2024-09-13T00:33:28.353885+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:11:25.868044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352536+0000","last_change":"2024-09-13T00:32:23.113059+0000","last_active":"2024-09-13T00:33:28.352536+0000","last_peered":"2024-09-13T00:33:28.352536+0000","last_clean":"2024-09-13T00:33:28.352536+0000","last_became_active":"2024-09-13T00:32:23.111858+0000","last_became_peered":"2024-09-13T00:32:23.111858+0000","last_unstale":"2024-09-13T00:33:28.352536+0000","last_undegraded":"2024-09-13T00:33:28.352536+0000","last_fullsized":"2024-09-13T00:33:28.352536+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T12:29:00.183110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.354590+0000","last_change":"2024-09-13T00:30:22.061674+0000","last_active":"2024-09-13T00:33:28.354590+0000","last_peered":"2024-09-13T00:33:28.354590+0000","last_clean":"2024-09-13T00:33:28.354590+0000","last_became_active":"2024-09-13T00:30:11.063404+0000","last_became_peered":"2024-09-13T00:30:11.063404+0000","last_unstale":"2024-09-13T00:33:28.354590+0000","last_undegraded":"2024-09-13T00:33:28.354590+0000","last_fullsized":"2024-09-13T00:33:28.354590+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:32:31.224759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053219400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.350987+0000","last_change":"2024-09-13T00:32:23.162665+0000","last_active":"2024-09-13T00:33:28.350987+0000","last_peered":"2024-09-13T00:33:28.350987+0000","last_clean":"2024-09-13T00:33:28.350987+0000","last_became_active":"2024-09-13T00:32:23.159005+0000","last_became_peered":"2024-09-13T00:32:23.159005+0000","last_unstale":"2024-09-13T00:33:28.350987+0000","last_undegraded":"2024-09-13T00:33:28.350987+0000","last_fullsized":"2024-09-13T00:33:28.350987+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:47:08.404863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.354318+0000","last_change":"2024-09-13T00:32:23.123503+0000","last_active":"2024-09-13T00:33:28.354318+0000","last_peered":"2024-09-13T00:33:28.354318+0000","last_clean":"2024-09-13T00:33:28.354318+0000","last_became_active":"2024-09-13T00:32:23.123111+0000","last_became_peered":"2024-09-13T00:32:23.123111+0000","last_unstale":"2024-09-13T00:33:28.354318+0000","last_undegraded":"2024-09-13T00:33:28.354318+0000","last_fullsized":"2024-09-13T00:33:28.354318+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T12:22:52.106632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.791421+0000","last_change":"2024-09-13T00:30:22.179990+0000","last_active":"2024-09-13T00:33:28.791421+0000","last_peered":"2024-09-13T00:33:28.791421+0000","last_clean":"2024-09-13T00:33:28.791421+0000","last_became_active":"2024-09-13T00:30:11.064377+0000","last_became_peered":"2024-09-13T00:30:11.064377+0000","last_unstale":"2024-09-13T00:33:28.791421+0000","last_undegraded":"2024-09-13T00:33:28.791421+0000","last_fullsized":"2024-09-13T00:33:28.791421+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:35:54.462078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077602300000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.346931+0000","last_change":"2024-09-13T00:32:23.860577+0000","last_active":"2024-09-13T00:33:28.346931+0000","last_peered":"2024-09-13T00:33:28.346931+0000","last_clean":"2024-09-13T00:33:28.346931+0000","last_became_active":"2024-09-13T00:32:23.859665+0000","last_became_peered":"2024-09-13T00:32:23.859665+0000","last_unstale":"2024-09-13T00:33:28.346931+0000","last_undegraded":"2024-09-13T00:33:28.346931+0000","last_fullsized":"2024-09-13T00:33:28.346931+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:28:41.285878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"62'2","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.355441+0000","last_change":"2024-09-13T00:32:23.114725+0000","last_active":"2024-09-13T00:33:28.355441+0000","last_peered":"2024-09-13T00:33:28.355441+0000","last_clean":"2024-09-13T00:33:28.355441+0000","last_became_active":"2024-09-13T00:32:23.113838+0000","last_became_peered":"2024-09-13T00:32:23.113838+0000","last_unstale":"2024-09-13T00:33:28.355441+0000","last_undegraded":"2024-09-13T00:33:28.355441+0000","last_fullsized":"2024-09-13T00:33:28.355441+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:06:02.251480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352554+0000","last_change":"2024-09-13T00:30:22.061997+0000","last_active":"2024-09-13T00:33:28.352554+0000","last_peered":"2024-09-13T00:33:28.352554+0000","last_clean":"2024-09-13T00:33:28.352554+0000","last_became_active":"2024-09-13T00:30:11.064525+0000","last_became_peered":"2024-09-13T00:30:11.064525+0000","last_unstale":"2024-09-13T00:33:28.352554+0000","last_undegraded":"2024-09-13T00:33:28.352554+0000","last_fullsized":"2024-09-13T00:33:28.352554+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:13:46.391207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085995799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352319+0000","last_change":"2024-09-13T00:32:23.861978+0000","last_active":"2024-09-13T00:33:28.352319+0000","last_peered":"2024-09-13T00:33:28.352319+0000","last_clean":"2024-09-13T00:33:28.352319+0000","last_became_active":"2024-09-13T00:32:23.861171+0000","last_became_peered":"2024-09-13T00:32:23.861171+0000","last_unstale":"2024-09-13T00:33:28.352319+0000","last_undegraded":"2024-09-13T00:33:28.352319+0000","last_fullsized":"2024-09-13T00:33:28.352319+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:57:49.312865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.347363+0000","last_change":"2024-09-13T00:32:23.133518+0000","last_active":"2024-09-13T00:33:28.347363+0000","last_peered":"2024-09-13T00:33:28.347363+0000","last_clean":"2024-09-13T00:33:28.347363+0000","last_became_active":"2024-09-13T00:32:23.132162+0000","last_became_peered":"2024-09-13T00:32:23.132162+0000","last_unstale":"2024-09-13T00:33:28.347363+0000","last_undegraded":"2024-09-13T00:33:28.347363+0000","last_fullsized":"2024-09-13T00:33:28.347363+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:24:53.942970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351705+0000","last_change":"2024-09-13T00:30:22.062239+0000","last_active":"2024-09-13T00:33:28.351705+0000","last_peered":"2024-09-13T00:33:28.351705+0000","last_clean":"2024-09-13T00:33:28.351705+0000","last_became_active":"2024-09-13T00:30:11.000458+0000","last_became_peered":"2024-09-13T00:30:11.000458+0000","last_unstale":"2024-09-13T00:33:28.351705+0000","last_undegraded":"2024-09-13T00:33:28.351705+0000","last_fullsized":"2024-09-13T00:33:28.351705+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:59:58.769064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00135603,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.345438+0000","last_change":"2024-09-13T00:32:23.151072+0000","last_active":"2024-09-13T00:33:28.345438+0000","last_peered":"2024-09-13T00:33:28.345438+0000","last_clean":"2024-09-13T00:33:28.345438+0000","last_became_active":"2024-09-13T00:32:23.150234+0000","last_became_peered":"2024-09-13T00:32:23.150234+0000","last_unstale":"2024-09-13T00:33:28.345438+0000","last_undegraded":"2024-09-13T00:33:28.345438+0000","last_fullsized":"2024-09-13T00:33:28.345438+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:03:02.780398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.347105+0000","last_change":"2024-09-13T00:32:23.160485+0000","last_active":"2024-09-13T00:33:28.347105+0000","last_peered":"2024-09-13T00:33:28.347105+0000","last_clean":"2024-09-13T00:33:28.347105+0000","last_became_active":"2024-09-13T00:32:23.158982+0000","last_became_peered":"2024-09-13T00:32:23.158982+0000","last_unstale":"2024-09-13T00:33:28.347105+0000","last_undegraded":"2024-09-13T00:33:28.347105+0000","last_fullsized":"2024-09-13T00:33:28.347105+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:54:03.153668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"63'7","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.345974+0000","last_change":"2024-09-13T00:32:23.100206+0000","last_active":"2024-09-13T00:33:28.345974+0000","last_peered":"2024-09-13T00:33:28.345974+0000","last_clean":"2024-09-13T00:33:28.345974+0000","last_became_active":"2024-09-13T00:32:23.098540+0000","last_became_peered":"2024-09-13T00:32:23.098540+0000","last_unstale":"2024-09-13T00:33:28.345974+0000","last_undegraded":"2024-09-13T00:33:28.345974+0000","last_fullsized":"2024-09-13T00:33:28.345974+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:20:56.437622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351075+0000","last_change":"2024-09-13T00:32:23.164292+0000","last_active":"2024-09-13T00:33:28.351075+0000","last_peered":"2024-09-13T00:33:28.351075+0000","last_clean":"2024-09-13T00:33:28.351075+0000","last_became_active":"2024-09-13T00:32:23.160050+0000","last_became_peered":"2024-09-13T00:32:23.160050+0000","last_unstale":"2024-09-13T00:33:28.351075+0000","last_undegraded":"2024-09-13T00:33:28.351075+0000","last_fullsized":"2024-09-13T00:33:28.351075+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:45:02.738945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"62'3","reported_seq":19,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.791297+0000","last_change":"2024-09-13T00:32:23.118269+0000","last_active":"2024-09-13T00:33:28.791297+0000","last_peered":"2024-09-13T00:33:28.791297+0000","last_clean":"2024-09-13T00:33:28.791297+0000","last_became_active":"2024-09-13T00:32:23.117466+0000","last_became_peered":"2024-09-13T00:32:23.117466+0000","last_unstale":"2024-09-13T00:33:28.791297+0000","last_undegraded":"2024-09-13T00:33:28.791297+0000","last_fullsized":"2024-09-13T00:33:28.791297+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:29:52.479917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351740+0000","last_change":"2024-09-13T00:32:23.126158+0000","last_active":"2024-09-13T00:33:28.351740+0000","last_peered":"2024-09-13T00:33:28.351740+0000","last_clean":"2024-09-13T00:33:28.351740+0000","last_became_active":"2024-09-13T00:32:23.125344+0000","last_became_peered":"2024-09-13T00:32:23.125344+0000","last_unstale":"2024-09-13T00:33:28.351740+0000","last_undegraded":"2024-09-13T00:33:28.351740+0000","last_fullsized":"2024-09-13T00:33:28.351740+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:06:33.501725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.353442+0000","last_change":"2024-09-13T00:32:23.864198+0000","last_active":"2024-09-13T00:33:28.353442+0000","last_peered":"2024-09-13T00:33:28.353442+0000","last_clean":"2024-09-13T00:33:28.353442+0000","last_became_active":"2024-09-13T00:32:23.863396+0000","last_became_peered":"2024-09-13T00:32:23.863396+0000","last_unstale":"2024-09-13T00:33:28.353442+0000","last_undegraded":"2024-09-13T00:33:28.353442+0000","last_fullsized":"2024-09-13T00:33:28.353442+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:05:47.048726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.346812+0000","last_change":"2024-09-13T00:32:23.129859+0000","last_active":"2024-09-13T00:33:28.346812+0000","last_peered":"2024-09-13T00:33:28.346812+0000","last_clean":"2024-09-13T00:33:28.346812+0000","last_became_active":"2024-09-13T00:32:23.129153+0000","last_became_peered":"2024-09-13T00:32:23.129153+0000","last_unstale":"2024-09-13T00:33:28.346812+0000","last_undegraded":"2024-09-13T00:33:28.346812+0000","last_fullsized":"2024-09-13T00:33:28.346812+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:07:04.367114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352236+0000","last_change":"2024-09-13T00:32:23.174513+0000","last_active":"2024-09-13T00:33:28.352236+0000","last_peered":"2024-09-13T00:33:28.352236+0000","last_clean":"2024-09-13T00:33:28.352236+0000","last_became_active":"2024-09-13T00:32:23.173751+0000","last_became_peered":"2024-09-13T00:32:23.173751+0000","last_unstale":"2024-09-13T00:33:28.352236+0000","last_undegraded":"2024-09-13T00:33:28.352236+0000","last_fullsized":"2024-09-13T00:33:28.352236+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:07:14.494788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351498+0000","last_change":"2024-09-13T00:32:23.162492+0000","last_active":"2024-09-13T00:33:28.351498+0000","last_peered":"2024-09-13T00:33:28.351498+0000","last_clean":"2024-09-13T00:33:28.351498+0000","last_became_active":"2024-09-13T00:32:23.162061+0000","last_became_peered":"2024-09-13T00:32:23.162061+0000","last_unstale":"2024-09-13T00:33:28.351498+0000","last_undegraded":"2024-09-13T00:33:28.351498+0000","last_fullsized":"2024-09-13T00:33:28.351498+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:49:36.897842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352410+0000","last_change":"2024-09-13T00:32:23.135038+0000","last_active":"2024-09-13T00:33:28.352410+0000","last_peered":"2024-09-13T00:33:28.352410+0000","last_clean":"2024-09-13T00:33:28.352410+0000","last_became_active":"2024-09-13T00:32:23.134333+0000","last_became_peered":"2024-09-13T00:32:23.134333+0000","last_unstale":"2024-09-13T00:33:28.352410+0000","last_undegraded":"2024-09-13T00:33:28.352410+0000","last_fullsized":"2024-09-13T00:33:28.352410+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:53:08.926676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.347022+0000","last_change":"2024-09-13T00:32:23.861211+0000","last_active":"2024-09-13T00:33:28.347022+0000","last_peered":"2024-09-13T00:33:28.347022+0000","last_clean":"2024-09-13T00:33:28.347022+0000","last_became_active":"2024-09-13T00:32:23.860309+0000","last_became_peered":"2024-09-13T00:32:23.860309+0000","last_unstale":"2024-09-13T00:33:28.347022+0000","last_undegraded":"2024-09-13T00:33:28.347022+0000","last_fullsized":"2024-09-13T00:33:28.347022+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:54:28.941726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351199+0000","last_change":"2024-09-13T00:32:23.131011+0000","last_active":"2024-09-13T00:33:28.351199+0000","last_peered":"2024-09-13T00:33:28.351199+0000","last_clean":"2024-09-13T00:33:28.351199+0000","last_became_active":"2024-09-13T00:32:23.130169+0000","last_became_peered":"2024-09-13T00:32:23.130169+0000","last_unstale":"2024-09-13T00:33:28.351199+0000","last_undegraded":"2024-09-13T00:33:28.351199+0000","last_fullsized":"2024-09-13T00:33:28.351199+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:28:55.512823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.355216+0000","last_change":"2024-09-13T00:32:23.172802+0000","last_active":"2024-09-13T00:33:28.355216+0000","last_peered":"2024-09-13T00:33:28.355216+0000","last_clean":"2024-09-13T00:33:28.355216+0000","last_became_active":"2024-09-13T00:32:23.172481+0000","last_became_peered":"2024-09-13T00:32:23.172481+0000","last_unstale":"2024-09-13T00:33:28.355216+0000","last_undegraded":"2024-09-13T00:33:28.355216+0000","last_fullsized":"2024-09-13T00:33:28.355216+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:59:15.071966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'2","reported_seq":16,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.345334+0000","last_change":"2024-09-13T00:32:23.102122+0000","last_active":"2024-09-13T00:33:28.345334+0000","last_peered":"2024-09-13T00:33:28.345334+0000","last_clean":"2024-09-13T00:33:28.345334+0000","last_became_active":"2024-09-13T00:32:23.098269+0000","last_became_peered":"2024-09-13T00:32:23.098269+0000","last_unstale":"2024-09-13T00:33:28.345334+0000","last_undegraded":"2024-09-13T00:33:28.345334+0000","last_fullsized":"2024-09-13T00:33:28.345334+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:33:52.642515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.346577+0000","last_change":"2024-09-13T00:32:23.165471+0000","last_active":"2024-09-13T00:33:28.346577+0000","last_peered":"2024-09-13T00:33:28.346577+0000","last_clean":"2024-09-13T00:33:28.346577+0000","last_became_active":"2024-09-13T00:32:23.164918+0000","last_became_peered":"2024-09-13T00:32:23.164918+0000","last_unstale":"2024-09-13T00:33:28.346577+0000","last_undegraded":"2024-09-13T00:33:28.346577+0000","last_fullsized":"2024-09-13T00:33:28.346577+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:20:24.972830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'2","reported_seq":16,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.347018+0000","last_change":"2024-09-13T00:32:23.129053+0000","last_active":"2024-09-13T00:33:28.347018+0000","last_peered":"2024-09-13T00:33:28.347018+0000","last_clean":"2024-09-13T00:33:28.347018+0000","last_became_active":"2024-09-13T00:32:23.128636+0000","last_became_peered":"2024-09-13T00:32:23.128636+0000","last_unstale":"2024-09-13T00:33:28.347018+0000","last_undegraded":"2024-09-13T00:33:28.347018+0000","last_fullsized":"2024-09-13T00:33:28.347018+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:09:01.573929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.346971+0000","last_change":"2024-09-13T00:32:23.167820+0000","last_active":"2024-09-13T00:33:28.346971+0000","last_peered":"2024-09-13T00:33:28.346971+0000","last_clean":"2024-09-13T00:33:28.346971+0000","last_became_active":"2024-09-13T00:32:23.166972+0000","last_became_peered":"2024-09-13T00:32:23.166972+0000","last_unstale":"2024-09-13T00:33:28.346971+0000","last_undegraded":"2024-09-13T00:33:28.346971+0000","last_fullsized":"2024-09-13T00:33:28.346971+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:21:47.508081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"62'2","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351911+0000","last_change":"2024-09-13T00:32:23.124131+0000","last_active":"2024-09-13T00:33:28.351911+0000","last_peered":"2024-09-13T00:33:28.351911+0000","last_clean":"2024-09-13T00:33:28.351911+0000","last_became_active":"2024-09-13T00:32:23.123871+0000","last_became_peered":"2024-09-13T00:32:23.123871+0000","last_unstale":"2024-09-13T00:33:28.351911+0000","last_undegraded":"2024-09-13T00:33:28.351911+0000","last_fullsized":"2024-09-13T00:33:28.351911+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:56:02.460621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.345202+0000","last_change":"2024-09-13T00:32:23.167683+0000","last_active":"2024-09-13T00:33:28.345202+0000","last_peered":"2024-09-13T00:33:28.345202+0000","last_clean":"2024-09-13T00:33:28.345202+0000","last_became_active":"2024-09-13T00:32:23.166379+0000","last_became_peered":"2024-09-13T00:32:23.166379+0000","last_unstale":"2024-09-13T00:33:28.345202+0000","last_undegraded":"2024-09-13T00:33:28.345202+0000","last_fullsized":"2024-09-13T00:33:28.345202+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:56:53.232461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"62'2","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352412+0000","last_change":"2024-09-13T00:32:23.118163+0000","last_active":"2024-09-13T00:33:28.352412+0000","last_peered":"2024-09-13T00:33:28.352412+0000","last_clean":"2024-09-13T00:33:28.352412+0000","last_became_active":"2024-09-13T00:32:23.117401+0000","last_became_peered":"2024-09-13T00:32:23.117401+0000","last_unstale":"2024-09-13T00:33:28.352412+0000","last_undegraded":"2024-09-13T00:33:28.352412+0000","last_fullsized":"2024-09-13T00:33:28.352412+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:17:55.385242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.354867+0000","last_change":"2024-09-13T00:32:23.166559+0000","last_active":"2024-09-13T00:33:28.354867+0000","last_peered":"2024-09-13T00:33:28.354867+0000","last_clean":"2024-09-13T00:33:28.354867+0000","last_became_active":"2024-09-13T00:32:23.165591+0000","last_became_peered":"2024-09-13T00:32:23.165591+0000","last_unstale":"2024-09-13T00:33:28.354867+0000","last_undegraded":"2024-09-13T00:33:28.354867+0000","last_fullsized":"2024-09-13T00:33:28.354867+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:21:34.838584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'2","reported_seq":16,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.355333+0000","last_change":"2024-09-13T00:32:23.117026+0000","last_active":"2024-09-13T00:33:28.355333+0000","last_peered":"2024-09-13T00:33:28.355333+0000","last_clean":"2024-09-13T00:33:28.355333+0000","last_became_active":"2024-09-13T00:32:23.116372+0000","last_became_peered":"2024-09-13T00:32:23.116372+0000","last_unstale":"2024-09-13T00:33:28.355333+0000","last_undegraded":"2024-09-13T00:33:28.355333+0000","last_fullsized":"2024-09-13T00:33:28.355333+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:02:40.393662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T00:33:29.815946+0000","last_change":"2024-09-13T00:33:29.815946+0000","last_active":"2024-09-13T00:33:29.815946+0000","last_peered":"2024-09-13T00:33:29.815946+0000","last_clean":"2024-09-13T00:33:29.815946+0000","last_became_active":"2024-09-13T00:33:29.815260+0000","last_became_peered":"2024-09-13T00:33:29.815260+0000","last_unstale":"2024-09-13T00:33:29.815946+0000","last_undegraded":"2024-09-13T00:33:29.815946+0000","last_fullsized":"2024-09-13T00:33:29.815946+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:05:55.292319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'5","reported_seq":21,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.346768+0000","last_change":"2024-09-13T00:32:23.127589+0000","last_active":"2024-09-13T00:33:28.346768+0000","last_peered":"2024-09-13T00:33:28.346768+0000","last_clean":"2024-09-13T00:33:28.346768+0000","last_became_active":"2024-09-13T00:32:23.126425+0000","last_became_peered":"2024-09-13T00:32:23.126425+0000","last_unstale":"2024-09-13T00:33:28.346768+0000","last_undegraded":"2024-09-13T00:33:28.346768+0000","last_fullsized":"2024-09-13T00:33:28.346768+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:32:44.249447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":47,"num_read_kb":49,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430279,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298616,"kb_used_data":1256,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446536,"statfs":{"total":95995035648,"available":95689252864,"internally_reserved":0,"allocated":1286144,"data_stored":886168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.89000000000000001,"15min":0.85999999999999999},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.1839999999999999,"5min":1.345,"15min":1.345},"last":0.94199999999999995},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.91200000000000003,"15min":0.874},"min":{"1min":0.67800000000000005,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.179,"5min":1.53,"15min":1.53},"last":0.98599999999999999}]},{"osd":1,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.84299999999999997,"15min":0.82499999999999996},"min":{"1min":0.54400000000000004,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.224,"5min":1.319,"15min":1.319},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.92700000000000005,"15min":0.86099999999999999},"min":{"1min":0.61099999999999999,"5min":0.435,"15min":0.435},"max":{"1min":1.1910000000000001,"5min":1.385,"15min":1.385},"last":0.90400000000000003}]},{"osd":2,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.93000000000000005,"15min":0.92100000000000004},"min":{"1min":0.70599999999999996,"5min":0.434,"15min":0.434},"max":{"1min":1.6100000000000001,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.91700000000000004,"15min":0.89700000000000002},"min":{"1min":0.57199999999999995,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.389,"5min":1.389,"15min":1.389},"last":0.88600000000000001}]},{"osd":3,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":0.94599999999999995,"15min":0.89100000000000001},"min":{"1min":0.64000000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.736,"5min":1.736,"15min":1.736},"last":1.0149999999999999},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.93899999999999995,"15min":0.93899999999999995},"min":{"1min":0.64500000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.177,"5min":1.306,"15min":1.306},"last":0.72199999999999998}]},{"osd":4,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.89000000000000001,"15min":0.89600000000000002},"min":{"1min":0.628,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.3129999999999999,"5min":1.4790000000000001,"15min":1.4790000000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.018,"5min":0.97899999999999998,"15min":0.90800000000000003},"min":{"1min":0.67500000000000004,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.5109999999999999,"5min":1.6160000000000001,"15min":1.6160000000000001},"last":0.749}]},{"osd":5,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.90300000000000002,"15min":0.88600000000000001},"min":{"1min":0.72899999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.1200000000000001,"5min":1.506,"15min":1.506},"last":0.83399999999999996},{"interface":"front","average":{"1min":1.02,"5min":0.98099999999999998,"15min":0.94399999999999995},"min":{"1min":0.72299999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.2829999999999999,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":0.81100000000000005}]},{"osd":6,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.0129999999999999,"15min":0.97699999999999998},"min":{"1min":0.85999999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.343,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":0.69499999999999995},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.0640000000000001,"15min":1.036},"min":{"1min":0.76600000000000001,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.8140000000000001,"5min":1.8140000000000001,"15min":1.8140000000000001},"last":1.0740000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626513,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298596,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446556,"statfs":{"total":95995035648,"available":95689273344,"internally_reserved":0,"allocated":1269760,"data_stored":885482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.88200000000000001,"15min":0.84199999999999997},"min":{"1min":0.51300000000000001,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":2.5659999999999998,"5min":2.5659999999999998,"15min":2.5659999999999998},"last":0.78900000000000003},{"interface":"front","average":{"1min":1.03,"5min":0.92200000000000004,"15min":0.873},"min":{"1min":0.65300000000000002,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":2.5310000000000001,"5min":2.5310000000000001,"15min":2.5310000000000001},"last":1.0009999999999999}]},{"osd":1,"last update":"Fri Sep 13 00:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.89700000000000002,"15min":0.91900000000000004},"min":{"1min":0.69799999999999995,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":2.5920000000000001,"5min":2.5920000000000001,"15min":2.5920000000000001},"last":1.0329999999999999},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.90100000000000002,"15min":0.89300000000000002},"min":{"1min":0.70399999999999996,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.2689999999999999,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":1.0600000000000001}]},{"osd":2,"last update":"Fri Sep 13 00:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.95899999999999996,"15min":0.91600000000000004},"min":{"1min":0.60299999999999998,"5min":0.443,"15min":0.443},"max":{"1min":1.24,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.0820000000000001},{"interface":"front","average":{"1min":1.032,"5min":0.91100000000000003,"15min":0.91800000000000004},"min":{"1min":0.71699999999999997,"5min":0.498,"15min":0.498},"max":{"1min":2.4889999999999999,"5min":2.4889999999999999,"15min":2.4889999999999999},"last":0.98499999999999999}]},{"osd":3,"last update":"Fri Sep 13 00:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.90600000000000003,"15min":0.91400000000000003},"min":{"1min":0.82799999999999996,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":1.121},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.0069999999999999,"15min":0.95899999999999996},"min":{"1min":0.73399999999999999,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.4339999999999999,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":1.214}]},{"osd":4,"last update":"Fri Sep 13 00:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.85099999999999998,"15min":0.83299999999999996},"min":{"1min":0.63600000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.121,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.94899999999999995,"15min":0.89100000000000001},"min":{"1min":0.69299999999999995,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.3169999999999999,"5min":1.403,"15min":1.403},"last":0.93700000000000006}]},{"osd":5,"last update":"Fri Sep 13 00:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.98199999999999998,"15min":0.874},"min":{"1min":0.69199999999999995,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.3540000000000001,"5min":1.357,"15min":1.357},"last":0.86899999999999999},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.97399999999999998,"15min":0.94199999999999995},"min":{"1min":0.84899999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.179,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":0.90100000000000002}]},{"osd":7,"last update":"Fri Sep 13 00:33:42 2024","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.0249999999999999,"15min":0.96899999999999997},"min":{"1min":0.80900000000000005,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.254,"5min":1.3959999999999999,"15min":1.3959999999999999},"last":0.74399999999999999},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.1359999999999999,"15min":1.0740000000000001},"min":{"1min":0.995,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.498,"5min":1.498,"15min":1.498},"last":1.153}]}]},{"osd":5,"up_from":36,"seq":154618822747,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":667648,"data_stored":294132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:33:08 2024","interfaces":[{"interface":"back","average":{"1min":0.84199999999999997,"5min":0.85999999999999999,"15min":0.82599999999999996},"min":{"1min":0.52000000000000002,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.1439999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":1.079},{"interface":"front","average":{"1min":0.98699999999999999,"5min":0.90800000000000003,"15min":0.85199999999999998},"min":{"1min":0.66200000000000003,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.2909999999999999,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.143}]},{"osd":1,"last update":"Fri Sep 13 00:33:08 2024","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.91400000000000003,"15min":0.84799999999999998},"min":{"1min":0.60399999999999998,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.216,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.86199999999999999,"5min":0.88700000000000001,"15min":0.85499999999999998},"min":{"1min":0.64400000000000002,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.1080000000000001,"5min":1.516,"15min":1.516},"last":1.171}]},{"osd":2,"last update":"Fri Sep 13 00:33:08 2024","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.85799999999999998,"15min":0.84399999999999997},"min":{"1min":0.63200000000000001,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.0660000000000001,"5min":1.306,"15min":1.306},"last":0.87},{"interface":"front","average":{"1min":0.95499999999999996,"5min":0.95999999999999996,"15min":0.86699999999999999},"min":{"1min":0.76700000000000002,"5min":0.62,"15min":0.53500000000000003},"max":{"1min":1.202,"5min":3.556,"15min":3.556},"last":1.105}]},{"osd":3,"last update":"Fri Sep 13 00:33:08 2024","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.92200000000000004,"15min":0.872},"min":{"1min":0.55500000000000005,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.139,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":0.99399999999999999},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.96599999999999997,"15min":0.871},"min":{"1min":0.626,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.28,"5min":1.456,"15min":1.456},"last":1.0129999999999999}]},{"osd":4,"last update":"Fri Sep 13 00:33:08 2024","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.84499999999999997,"15min":0.70199999999999996},"min":{"1min":0.40400000000000003,"5min":0.376,"15min":0.34499999999999997},"max":{"1min":1.081,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":1.262},{"interface":"front","average":{"1min":0.878,"5min":0.91500000000000004,"15min":0.745},"min":{"1min":0.49199999999999999,"5min":0.375,"15min":0.30599999999999999},"max":{"1min":1.155,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":0.77300000000000002}]},{"osd":6,"last update":"Fri Sep 13 00:33:51 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.93700000000000006,"15min":0.871},"min":{"1min":0.65200000000000002,"5min":0.44600000000000001,"15min":0.41399999999999998},"max":{"1min":1.206,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.073,"15min":0.96699999999999997},"min":{"1min":0.72799999999999998,"5min":0.45700000000000002,"15min":0.45300000000000001},"max":{"1min":1.2889999999999999,"5min":1.603,"15min":1.603},"last":1.1970000000000001}]},{"osd":7,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.1000000000000001,"15min":1.0880000000000001},"min":{"1min":0.81100000000000005,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.3560000000000001,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.236},{"interface":"front","average":{"1min":1.0349999999999999,"5min":0.995,"15min":1.0229999999999999},"min":{"1min":0.73199999999999998,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.248,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.2170000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018980,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298032,"kb_used_data":672,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447120,"statfs":{"total":95995035648,"available":95689850880,"internally_reserved":0,"allocated":688128,"data_stored":295730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.90400000000000003,"15min":0.81200000000000006},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.0920000000000001,"5min":1.216,"15min":1.216},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.89400000000000002,"5min":0.91100000000000003,"15min":0.83299999999999996},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.149,"5min":1.284,"15min":1.284},"last":0.872}]},{"osd":1,"last update":"Fri Sep 13 00:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.90600000000000003,"15min":0.83199999999999996},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.2529999999999999,"5min":1.405,"15min":1.405},"last":0.92000000000000004},{"interface":"front","average":{"1min":0.93700000000000006,"5min":0.93600000000000005,"15min":0.85199999999999998},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.438},"max":{"1min":1.2250000000000001,"5min":1.2509999999999999,"15min":1.2509999999999999},"last":0.95999999999999996}]},{"osd":2,"last update":"Fri Sep 13 00:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":0.92000000000000004,"15min":0.84899999999999998},"min":{"1min":0.54600000000000004,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.0980000000000001,"5min":1.196,"15min":1.196},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.90200000000000002,"15min":0.84599999999999997},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.48399999999999999},"max":{"1min":1.1479999999999999,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":1.1699999999999999}]},{"osd":3,"last update":"Fri Sep 13 00:33:17 2024","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.91100000000000003,"15min":0.90100000000000002},"min":{"1min":0.58599999999999997,"5min":0.52700000000000002,"15min":0.46600000000000003},"max":{"1min":1.1970000000000001,"5min":1.575,"15min":1.587},"last":0.98699999999999999},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.95099999999999996,"15min":0.89200000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.2430000000000001,"5min":1.302,"15min":1.302},"last":1.014}]},{"osd":5,"last update":"Fri Sep 13 00:34:07 2024","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.92100000000000004,"15min":0.73699999999999999},"min":{"1min":0.63500000000000001,"5min":0.315,"15min":0.315},"max":{"1min":1.149,"5min":1.405,"15min":1.405},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.93000000000000005,"15min":0.76500000000000001},"min":{"1min":0.68300000000000005,"5min":0.35199999999999998,"15min":0.33100000000000002},"max":{"1min":1.1499999999999999,"5min":1.472,"15min":1.472},"last":0.82199999999999995}]},{"osd":6,"last update":"Fri Sep 13 00:33:48 2024","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":0.95699999999999996,"15min":0.84799999999999998},"min":{"1min":0.64000000000000001,"5min":0.39100000000000001,"15min":0.34499999999999997},"max":{"1min":1.2430000000000001,"5min":1.347,"15min":1.347},"last":1.0649999999999999},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.016,"15min":0.91200000000000003},"min":{"1min":0.82099999999999995,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.294,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":1.046}]},{"osd":7,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.999,"15min":1.0209999999999999},"min":{"1min":0.69599999999999995,"5min":0.37,"15min":0.37},"max":{"1min":1.216,"5min":1.8480000000000001,"15min":1.8480000000000001},"last":1.0900000000000001},{"interface":"front","average":{"1min":1.121,"5min":1.0429999999999999,"15min":1.026},"min":{"1min":0.749,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.395,"5min":1.464,"15min":1.464},"last":1.129}]}]},{"osd":3,"up_from":25,"seq":107374182510,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":667648,"data_stored":294122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:33:25 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.66200000000000003,"15min":0.64700000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.91700000000000004,"5min":1.175,"15min":1.175},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.748,"5min":0.72699999999999998,"15min":0.747},"min":{"1min":0.32500000000000001,"5min":0.307,"15min":0.307},"max":{"1min":1.0669999999999999,"5min":1.1619999999999999,"15min":2.1400000000000001},"last":0.93999999999999995}]},{"osd":1,"last update":"Fri Sep 13 00:33:25 2024","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.755,"15min":0.76400000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.0589999999999999,"5min":1.1319999999999999,"15min":2.2029999999999998},"last":0.80000000000000004},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.72099999999999997,"15min":0.71099999999999997},"min":{"1min":0.34399999999999997,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.0289999999999999,"5min":1.274,"15min":1.274},"last":0.69599999999999995}]},{"osd":2,"last update":"Fri Sep 13 00:33:25 2024","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.77600000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":1.4299999999999999,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.75700000000000001,"15min":0.72099999999999997},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.212,"5min":1.212,"15min":1.212},"last":0.73299999999999998}]},{"osd":4,"last update":"Fri Sep 13 00:33:18 2024","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.91100000000000003,"15min":0.88800000000000001},"min":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.1850000000000001,"5min":1.2410000000000001,"15min":1.2410000000000001},"last":0.83499999999999996},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.98699999999999999,"15min":0.91500000000000004},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.2909999999999999,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.0129999999999999}]},{"osd":5,"last update":"Fri Sep 13 00:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.97599999999999998,"15min":0.95799999999999996},"min":{"1min":0.63600000000000001,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.2909999999999999,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":1.175},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.97499999999999998,"15min":0.95199999999999996},"min":{"1min":0.67500000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.472,"5min":1.732,"15min":1.732},"last":0.96499999999999997}]},{"osd":6,"last update":"Fri Sep 13 00:33:51 2024","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.022,"15min":1.0840000000000001},"min":{"1min":0.66100000000000003,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.26,"5min":1.4570000000000001,"15min":2.6040000000000001},"last":1.0800000000000001},{"interface":"front","average":{"1min":0.999,"5min":1.014,"15min":1.03},"min":{"1min":0.61099999999999999,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":1.302,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":0.91000000000000003}]},{"osd":7,"last update":"Fri Sep 13 00:33:41 2024","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.04,"15min":1.097},"min":{"1min":0.68899999999999995,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.417,"5min":1.619,"15min":1.619},"last":1.036},{"interface":"front","average":{"1min":1.024,"5min":1.018,"15min":1.0740000000000001},"min":{"1min":0.623,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.3799999999999999,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":1.1359999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411450,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298012,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447140,"statfs":{"total":95995035648,"available":95689871360,"internally_reserved":0,"allocated":671744,"data_stored":294636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:33:32 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.67300000000000004,"15min":0.58999999999999997},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":1.125,"5min":1.125,"15min":1.125},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.67700000000000005,"15min":0.59899999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.98099999999999998,"5min":1.113,"15min":1.113},"last":0.59699999999999998}]},{"osd":1,"last update":"Fri Sep 13 00:33:32 2024","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":0.73899999999999999,"15min":0.66200000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33800000000000002},"max":{"1min":1.1990000000000001,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.86799999999999999},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.755,"15min":0.65000000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":1.101,"5min":1.8680000000000001,"15min":1.8680000000000001},"last":0.79300000000000004}]},{"osd":3,"last update":"Fri Sep 13 00:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":0.82099999999999995,"15min":0.77000000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.2110000000000001,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":0.95899999999999996},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.82099999999999995,"15min":0.80300000000000005},"min":{"1min":0.38300000000000001,"5min":0.377,"15min":0.377},"max":{"1min":1.2909999999999999,"5min":1.3049999999999999,"15min":1.319},"last":1.0249999999999999}]},{"osd":4,"last update":"Fri Sep 13 00:33:24 2024","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.98399999999999999,"15min":0.93300000000000005},"min":{"1min":0.71699999999999997,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.25,"5min":1.395,"15min":1.395},"last":1.0940000000000001},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.0409999999999999,"15min":0.94599999999999995},"min":{"1min":0.73799999999999999,"5min":0.71599999999999997,"15min":0.71599999999999997},"max":{"1min":1.3129999999999999,"5min":1.6599999999999999,"15min":1.6599999999999999},"last":1.169}]},{"osd":5,"last update":"Fri Sep 13 00:33:07 2024","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":0.98699999999999999,"15min":0.95599999999999996},"min":{"1min":0.76900000000000002,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.5900000000000001,"5min":1.5900000000000001,"15min":1.5900000000000001},"last":1.0649999999999999},{"interface":"front","average":{"1min":0.97699999999999998,"5min":0.95099999999999996,"15min":0.91200000000000003},"min":{"1min":0.625,"5min":0.625,"15min":0.625},"max":{"1min":1.5509999999999999,"5min":1.5509999999999999,"15min":1.5509999999999999},"last":0.89000000000000001}]},{"osd":6,"last update":"Fri Sep 13 00:33:53 2024","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":0.97899999999999998,"15min":0.98599999999999999},"min":{"1min":0.63500000000000001,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.2589999999999999,"5min":1.641,"15min":1.641},"last":0.996},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.0289999999999999,"15min":1.0249999999999999},"min":{"1min":0.752,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.163,"5min":1.51,"15min":1.51},"last":1.123}]},{"osd":7,"last update":"Fri Sep 13 00:33:36 2024","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.0640000000000001,"15min":1.0780000000000001},"min":{"1min":0.81699999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"max":{"1min":1.3759999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":1.2130000000000001},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.0509999999999999,"15min":1.0800000000000001},"min":{"1min":0.56699999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.4730000000000001,"5min":1.6220000000000001,"15min":1.6220000000000001},"last":0.92600000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574982,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298012,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447140,"statfs":{"total":95995035648,"available":95689871360,"internally_reserved":0,"allocated":667648,"data_stored":294051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:33:51 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.63800000000000001,"15min":0.56100000000000005},"min":{"1min":0.307,"5min":0.307,"15min":0.25900000000000001},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.76700000000000002,"15min":0.64800000000000002},"min":{"1min":0.47799999999999998,"5min":0.30099999999999999,"15min":0.28299999999999997},"max":{"1min":1.7170000000000001,"5min":1.7170000000000001,"15min":1.8879999999999999},"last":0.65100000000000002}]},{"osd":2,"last update":"Fri Sep 13 00:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.75600000000000001,"15min":0.64400000000000002},"min":{"1min":0.45400000000000001,"5min":0.36199999999999999,"15min":0.28699999999999998},"max":{"1min":1.825,"5min":1.825,"15min":1.825},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.75,"5min":0.70599999999999996,"15min":0.63},"min":{"1min":0.51700000000000002,"5min":0.33100000000000002,"15min":0.32000000000000001},"max":{"1min":1.21,"5min":1.21,"15min":1.21},"last":1.012}]},{"osd":3,"last update":"Fri Sep 13 00:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.755,"15min":0.72099999999999997},"min":{"1min":0.63700000000000001,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.0980000000000001,"5min":1.0980000000000001,"15min":1.0980000000000001},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.85299999999999998,"15min":0.79600000000000004},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.8759999999999999,"5min":1.8759999999999999,"15min":1.8759999999999999},"last":0.92700000000000005}]},{"osd":4,"last update":"Fri Sep 13 00:33:20 2024","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.0429999999999999,"15min":0.96699999999999997},"min":{"1min":0.84099999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.2629999999999999,"5min":1.7809999999999999,"15min":1.7809999999999999},"last":1.1299999999999999},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.97799999999999998,"15min":0.88800000000000001},"min":{"1min":0.77900000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.359,"5min":1.633,"15min":1.633},"last":1.1080000000000001}]},{"osd":5,"last update":"Fri Sep 13 00:34:05 2024","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.91500000000000004,"15min":0.91000000000000003},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.181,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.94799999999999995,"15min":0.92800000000000005},"min":{"1min":0.621,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.1930000000000001,"5min":1.286,"15min":1.371},"last":1.0509999999999999}]},{"osd":6,"last update":"Fri Sep 13 00:33:58 2024","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.0469999999999999,"15min":1.0569999999999999},"min":{"1min":0.69599999999999995,"5min":0.625,"15min":0.625},"max":{"1min":1.921,"5min":1.921,"15min":1.921},"last":1.2330000000000001},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.01,"15min":1.002},"min":{"1min":0.77300000000000002,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.2210000000000001,"5min":1.823,"15min":1.823},"last":1.167}]},{"osd":7,"last update":"Fri Sep 13 00:33:51 2024","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"min":{"1min":0.70999999999999996,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.2330000000000001,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":1.2110000000000001},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.111,"15min":1.0920000000000001},"min":{"1min":0.66500000000000004,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":2.0070000000000001,"5min":2.0070000000000001,"15min":2.0070000000000001},"last":1.1970000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738511,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298596,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446556,"statfs":{"total":95995035648,"available":95689273344,"internally_reserved":0,"allocated":1265664,"data_stored":884992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 00:33:32 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.60999999999999999,"15min":0.54400000000000004},"min":{"1min":0.32600000000000001,"5min":0.317,"15min":0.24399999999999999},"max":{"1min":0.80800000000000005,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68600000000000005,"15min":0.58499999999999996},"min":{"1min":0.38300000000000001,"5min":0.28199999999999997,"15min":0.27400000000000002},"max":{"1min":0.92400000000000004,"5min":1.1799999999999999,"15min":1.2050000000000001},"last":0.70699999999999996}]},{"osd":2,"last update":"Fri Sep 13 00:33:32 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.60599999999999998},"min":{"1min":0.313,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.05,"5min":1.246,"15min":1.246},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.81599999999999995,"5min":0.82499999999999996,"15min":0.73399999999999999},"min":{"1min":0.441,"5min":0.312,"15min":0.312},"max":{"1min":0.98899999999999999,"5min":1.216,"15min":1.216},"last":0.92200000000000004}]},{"osd":3,"last update":"Fri Sep 13 00:33:21 2024","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.76800000000000002,"15min":0.73899999999999999},"min":{"1min":0.36399999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.1259999999999999,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.89600000000000002,"15min":0.81399999999999995},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":1.0489999999999999}]},{"osd":4,"last update":"Fri Sep 13 00:33:17 2024","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.0269999999999999,"15min":0.92500000000000004},"min":{"1min":0.72299999999999998,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.401,"5min":1.573,"15min":1.573},"last":1.097},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.89200000000000002},"min":{"1min":0.58399999999999996,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.1000000000000001,"5min":1.9199999999999999,"15min":1.9199999999999999},"last":0.96499999999999997}]},{"osd":5,"last update":"Fri Sep 13 00:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.876,"15min":0.86199999999999999},"min":{"1min":0.66200000000000003,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.101,"5min":1.272,"15min":1.272},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.98699999999999999,"15min":0.98199999999999998},"min":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.296,"5min":1.5,"15min":1.5},"last":1.139}]},{"osd":6,"last update":"Fri Sep 13 00:33:51 2024","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.0209999999999999,"15min":1.0389999999999999},"min":{"1min":0.65600000000000003,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.3440000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":0.97799999999999998},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.97599999999999998,"15min":0.94399999999999995},"min":{"1min":0.52900000000000003,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.208,"5min":1.99,"15min":1.99},"last":1.0089999999999999}]},{"osd":7,"last update":"Fri Sep 13 00:33:42 2024","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.083,"15min":1.119},"min":{"1min":0.64500000000000002,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.278,"5min":1.6319999999999999,"15min":1.6319999999999999},"last":1.1850000000000001},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.0049999999999999,"15min":1.0449999999999999},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.2330000000000001,"5min":1.8280000000000001,"15min":1.8280000000000001},"last":1.1200000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T00:34:14.283 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T00:34:14.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:13 smithi088 bash[24895]: audit 2024-09-13T00:34:13.479039+0000 mgr.y (mgr.14397) 182 : audit [DBG] from='client.24620 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:34:15.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:14 smithi077 bash[22596]: cluster 2024-09-13T00:34:13.842174+0000 mgr.y (mgr.14397) 183 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:15.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:14 smithi077 bash[32040]: cluster 2024-09-13T00:34:13.842174+0000 mgr.y (mgr.14397) 183 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:15.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:14 smithi088 bash[24895]: cluster 2024-09-13T00:34:13.842174+0000 mgr.y (mgr.14397) 183 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:17.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:16 smithi077 bash[22596]: cluster 2024-09-13T00:34:15.842780+0000 mgr.y (mgr.14397) 184 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:17.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:16 smithi077 bash[32040]: cluster 2024-09-13T00:34:15.842780+0000 mgr.y (mgr.14397) 184 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:17.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:16 smithi088 bash[24895]: cluster 2024-09-13T00:34:15.842780+0000 mgr.y (mgr.14397) 184 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:19.066 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:34:19.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:18 smithi077 bash[22596]: cluster 2024-09-13T00:34:17.843333+0000 mgr.y (mgr.14397) 185 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:19.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:18 smithi077 bash[32040]: cluster 2024-09-13T00:34:17.843333+0000 mgr.y (mgr.14397) 185 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:19.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:18 smithi088 bash[24895]: cluster 2024-09-13T00:34:17.843333+0000 mgr.y (mgr.14397) 185 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:21.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:20 smithi077 bash[22596]: cluster 2024-09-13T00:34:19.844399+0000 mgr.y (mgr.14397) 186 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:21.025 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:20 smithi077 bash[32040]: cluster 2024-09-13T00:34:19.844399+0000 mgr.y (mgr.14397) 186 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:21.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:20 smithi088 bash[24895]: cluster 2024-09-13T00:34:19.844399+0000 mgr.y (mgr.14397) 186 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:21.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:34:20 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:34:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:34:21.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:34:21 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:34:21] "GET /metrics HTTP/1.1" 200 240026 "" "Prometheus/2.43.0" 2024-09-13T00:34:22.845 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:34:22.846 INFO:teuthology.orchestra.run.smithi077.stderr:dumped all 2024-09-13T00:34:23.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:22 smithi077 bash[22596]: cluster 2024-09-13T00:34:21.845045+0000 mgr.y (mgr.14397) 187 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:23.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:22 smithi077 bash[32040]: cluster 2024-09-13T00:34:21.845045+0000 mgr.y (mgr.14397) 187 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:23.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:22 smithi088 bash[24895]: cluster 2024-09-13T00:34:21.845045+0000 mgr.y (mgr.14397) 187 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:23.614 INFO:teuthology.orchestra.run.smithi077.stdout:{"pg_ready":true,"pg_map":{"version":164,"stamp":"2024-09-13T00:34:21.844735+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592725,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":173,"num_read_kb":280,"num_write":160,"num_write_kb":1643,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127,"ondisk_log_size":127,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385880,"kb_used_data":7016,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575336,"statfs":{"total":767960285184,"available":765517144064,"internally_reserved":0,"allocated":7184384,"data_stored":4129313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004717"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.355068+0000","last_change":"2024-09-13T00:32:23.860928+0000","last_active":"2024-09-13T00:33:28.355068+0000","last_peered":"2024-09-13T00:33:28.355068+0000","last_clean":"2024-09-13T00:33:28.355068+0000","last_became_active":"2024-09-13T00:32:23.860092+0000","last_became_peered":"2024-09-13T00:32:23.860092+0000","last_unstale":"2024-09-13T00:33:28.355068+0000","last_undegraded":"2024-09-13T00:33:28.355068+0000","last_fullsized":"2024-09-13T00:33:28.355068+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:26:04.784182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"62'2","reported_seq":18,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.790845+0000","last_change":"2024-09-13T00:32:23.110415+0000","last_active":"2024-09-13T00:33:28.790845+0000","last_peered":"2024-09-13T00:33:28.790845+0000","last_clean":"2024-09-13T00:33:28.790845+0000","last_became_active":"2024-09-13T00:32:23.109400+0000","last_became_peered":"2024-09-13T00:32:23.109400+0000","last_unstale":"2024-09-13T00:33:28.790845+0000","last_undegraded":"2024-09-13T00:33:28.790845+0000","last_fullsized":"2024-09-13T00:33:28.790845+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:04:15.530283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352278+0000","last_change":"2024-09-13T00:32:23.861242+0000","last_active":"2024-09-13T00:33:28.352278+0000","last_peered":"2024-09-13T00:33:28.352278+0000","last_clean":"2024-09-13T00:33:28.352278+0000","last_became_active":"2024-09-13T00:32:23.860422+0000","last_became_peered":"2024-09-13T00:32:23.860422+0000","last_unstale":"2024-09-13T00:33:28.352278+0000","last_undegraded":"2024-09-13T00:33:28.352278+0000","last_fullsized":"2024-09-13T00:33:28.352278+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:34:40.070133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.345573+0000","last_change":"2024-09-13T00:32:23.128078+0000","last_active":"2024-09-13T00:33:28.345573+0000","last_peered":"2024-09-13T00:33:28.345573+0000","last_clean":"2024-09-13T00:33:28.345573+0000","last_became_active":"2024-09-13T00:32:23.127612+0000","last_became_peered":"2024-09-13T00:32:23.127612+0000","last_unstale":"2024-09-13T00:33:28.345573+0000","last_undegraded":"2024-09-13T00:33:28.345573+0000","last_fullsized":"2024-09-13T00:33:28.345573+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:36:20.193555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.347496+0000","last_change":"2024-09-13T00:32:23.863860+0000","last_active":"2024-09-13T00:33:28.347496+0000","last_peered":"2024-09-13T00:33:28.347496+0000","last_clean":"2024-09-13T00:33:28.347496+0000","last_became_active":"2024-09-13T00:32:23.861277+0000","last_became_peered":"2024-09-13T00:32:23.861277+0000","last_unstale":"2024-09-13T00:33:28.347496+0000","last_undegraded":"2024-09-13T00:33:28.347496+0000","last_fullsized":"2024-09-13T00:33:28.347496+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:32:46.629677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.790534+0000","last_change":"2024-09-13T00:32:23.110220+0000","last_active":"2024-09-13T00:33:28.790534+0000","last_peered":"2024-09-13T00:33:28.790534+0000","last_clean":"2024-09-13T00:33:28.790534+0000","last_became_active":"2024-09-13T00:32:23.109022+0000","last_became_peered":"2024-09-13T00:32:23.109022+0000","last_unstale":"2024-09-13T00:33:28.790534+0000","last_undegraded":"2024-09-13T00:33:28.790534+0000","last_fullsized":"2024-09-13T00:33:28.790534+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:23:26.497941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.355372+0000","last_change":"2024-09-13T00:32:23.860610+0000","last_active":"2024-09-13T00:33:28.355372+0000","last_peered":"2024-09-13T00:33:28.355372+0000","last_clean":"2024-09-13T00:33:28.355372+0000","last_became_active":"2024-09-13T00:32:23.859814+0000","last_became_peered":"2024-09-13T00:32:23.859814+0000","last_unstale":"2024-09-13T00:33:28.355372+0000","last_undegraded":"2024-09-13T00:33:28.355372+0000","last_fullsized":"2024-09-13T00:33:28.355372+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:06:43.248402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"62'8","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.353028+0000","last_change":"2024-09-13T00:32:23.122582+0000","last_active":"2024-09-13T00:33:28.353028+0000","last_peered":"2024-09-13T00:33:28.353028+0000","last_clean":"2024-09-13T00:33:28.353028+0000","last_became_active":"2024-09-13T00:32:23.121812+0000","last_became_peered":"2024-09-13T00:32:23.121812+0000","last_unstale":"2024-09-13T00:33:28.353028+0000","last_undegraded":"2024-09-13T00:33:28.353028+0000","last_fullsized":"2024-09-13T00:33:28.353028+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:29:49.937607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"62'1","reported_seq":16,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.791359+0000","last_change":"2024-09-13T00:32:23.121516+0000","last_active":"2024-09-13T00:33:28.791359+0000","last_peered":"2024-09-13T00:33:28.791359+0000","last_clean":"2024-09-13T00:33:28.791359+0000","last_became_active":"2024-09-13T00:32:23.120641+0000","last_became_peered":"2024-09-13T00:32:23.120641+0000","last_unstale":"2024-09-13T00:33:28.791359+0000","last_undegraded":"2024-09-13T00:33:28.791359+0000","last_fullsized":"2024-09-13T00:33:28.791359+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:35:52.434687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.350681+0000","last_change":"2024-09-13T00:32:23.861007+0000","last_active":"2024-09-13T00:33:28.350681+0000","last_peered":"2024-09-13T00:33:28.350681+0000","last_clean":"2024-09-13T00:33:28.350681+0000","last_became_active":"2024-09-13T00:32:23.860213+0000","last_became_peered":"2024-09-13T00:32:23.860213+0000","last_unstale":"2024-09-13T00:33:28.350681+0000","last_undegraded":"2024-09-13T00:33:28.350681+0000","last_fullsized":"2024-09-13T00:33:28.350681+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:58:20.866610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"62'2","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.791012+0000","last_change":"2024-09-13T00:32:23.109098+0000","last_active":"2024-09-13T00:33:28.791012+0000","last_peered":"2024-09-13T00:33:28.791012+0000","last_clean":"2024-09-13T00:33:28.791012+0000","last_became_active":"2024-09-13T00:32:23.108209+0000","last_became_peered":"2024-09-13T00:32:23.108209+0000","last_unstale":"2024-09-13T00:33:28.791012+0000","last_undegraded":"2024-09-13T00:33:28.791012+0000","last_fullsized":"2024-09-13T00:33:28.791012+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:55:33.433874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":23,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T00:33:29.788338+0000","last_change":"2024-09-13T00:33:29.788338+0000","last_active":"2024-09-13T00:33:29.788338+0000","last_peered":"2024-09-13T00:33:29.788338+0000","last_clean":"2024-09-13T00:33:29.788338+0000","last_became_active":"2024-09-13T00:33:29.787460+0000","last_became_peered":"2024-09-13T00:33:29.787460+0000","last_unstale":"2024-09-13T00:33:29.788338+0000","last_undegraded":"2024-09-13T00:33:29.788338+0000","last_fullsized":"2024-09-13T00:33:29.788338+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:17:54.554758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"62'1","reported_seq":16,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.791312+0000","last_change":"2024-09-13T00:32:23.122340+0000","last_active":"2024-09-13T00:33:28.791312+0000","last_peered":"2024-09-13T00:33:28.791312+0000","last_clean":"2024-09-13T00:33:28.791312+0000","last_became_active":"2024-09-13T00:32:23.121795+0000","last_became_peered":"2024-09-13T00:32:23.121795+0000","last_unstale":"2024-09-13T00:33:28.791312+0000","last_undegraded":"2024-09-13T00:33:28.791312+0000","last_fullsized":"2024-09-13T00:33:28.791312+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:18:28.661306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.350314+0000","last_change":"2024-09-13T00:32:23.861336+0000","last_active":"2024-09-13T00:33:28.350314+0000","last_peered":"2024-09-13T00:33:28.350314+0000","last_clean":"2024-09-13T00:33:28.350314+0000","last_became_active":"2024-09-13T00:32:23.860083+0000","last_became_peered":"2024-09-13T00:32:23.860083+0000","last_unstale":"2024-09-13T00:33:28.350314+0000","last_undegraded":"2024-09-13T00:33:28.350314+0000","last_fullsized":"2024-09-13T00:33:28.350314+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:18:50.519864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352085+0000","last_change":"2024-09-13T00:32:23.118671+0000","last_active":"2024-09-13T00:33:28.352085+0000","last_peered":"2024-09-13T00:33:28.352085+0000","last_clean":"2024-09-13T00:33:28.352085+0000","last_became_active":"2024-09-13T00:32:23.113524+0000","last_became_peered":"2024-09-13T00:32:23.113524+0000","last_unstale":"2024-09-13T00:33:28.352085+0000","last_undegraded":"2024-09-13T00:33:28.352085+0000","last_fullsized":"2024-09-13T00:33:28.352085+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:08:35.149880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":23,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T00:33:29.364966+0000","last_change":"2024-09-13T00:33:29.364966+0000","last_active":"2024-09-13T00:33:29.364966+0000","last_peered":"2024-09-13T00:33:29.364966+0000","last_clean":"2024-09-13T00:33:29.364966+0000","last_became_active":"2024-09-13T00:33:29.364191+0000","last_became_peered":"2024-09-13T00:33:29.364191+0000","last_unstale":"2024-09-13T00:33:29.364966+0000","last_undegraded":"2024-09-13T00:33:29.364966+0000","last_fullsized":"2024-09-13T00:33:29.364966+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:45:42.373334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"63'3","reported_seq":18,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.354637+0000","last_change":"2024-09-13T00:32:23.118698+0000","last_active":"2024-09-13T00:33:28.354637+0000","last_peered":"2024-09-13T00:33:28.354637+0000","last_clean":"2024-09-13T00:33:28.354637+0000","last_became_active":"2024-09-13T00:32:23.118060+0000","last_became_peered":"2024-09-13T00:32:23.118060+0000","last_unstale":"2024-09-13T00:33:28.354637+0000","last_undegraded":"2024-09-13T00:33:28.354637+0000","last_fullsized":"2024-09-13T00:33:28.354637+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:32:06.783542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.345276+0000","last_change":"2024-09-13T00:32:23.168009+0000","last_active":"2024-09-13T00:33:28.345276+0000","last_peered":"2024-09-13T00:33:28.345276+0000","last_clean":"2024-09-13T00:33:28.345276+0000","last_became_active":"2024-09-13T00:32:23.167303+0000","last_became_peered":"2024-09-13T00:32:23.167303+0000","last_unstale":"2024-09-13T00:33:28.345276+0000","last_undegraded":"2024-09-13T00:33:28.345276+0000","last_fullsized":"2024-09-13T00:33:28.345276+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:45:27.608084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.350684+0000","last_change":"2024-09-13T00:32:23.096317+0000","last_active":"2024-09-13T00:33:28.350684+0000","last_peered":"2024-09-13T00:33:28.350684+0000","last_clean":"2024-09-13T00:33:28.350684+0000","last_became_active":"2024-09-13T00:32:23.095610+0000","last_became_peered":"2024-09-13T00:32:23.095610+0000","last_unstale":"2024-09-13T00:33:28.350684+0000","last_undegraded":"2024-09-13T00:33:28.350684+0000","last_fullsized":"2024-09-13T00:33:28.350684+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:32:37.463877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.346418+0000","last_change":"2024-09-13T00:32:23.160451+0000","last_active":"2024-09-13T00:33:28.346418+0000","last_peered":"2024-09-13T00:33:28.346418+0000","last_clean":"2024-09-13T00:33:28.346418+0000","last_became_active":"2024-09-13T00:32:23.158923+0000","last_became_peered":"2024-09-13T00:32:23.158923+0000","last_unstale":"2024-09-13T00:33:28.346418+0000","last_undegraded":"2024-09-13T00:33:28.346418+0000","last_fullsized":"2024-09-13T00:33:28.346418+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:07:10.841651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.345665+0000","last_change":"2024-09-13T00:32:23.103760+0000","last_active":"2024-09-13T00:33:28.345665+0000","last_peered":"2024-09-13T00:33:28.345665+0000","last_clean":"2024-09-13T00:33:28.345665+0000","last_became_active":"2024-09-13T00:32:23.102653+0000","last_became_peered":"2024-09-13T00:32:23.102653+0000","last_unstale":"2024-09-13T00:33:28.345665+0000","last_undegraded":"2024-09-13T00:33:28.345665+0000","last_fullsized":"2024-09-13T00:33:28.345665+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:10:24.285084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.346342+0000","last_change":"2024-09-13T00:32:23.159864+0000","last_active":"2024-09-13T00:33:28.346342+0000","last_peered":"2024-09-13T00:33:28.346342+0000","last_clean":"2024-09-13T00:33:28.346342+0000","last_became_active":"2024-09-13T00:32:23.158975+0000","last_became_peered":"2024-09-13T00:32:23.158975+0000","last_unstale":"2024-09-13T00:33:28.346342+0000","last_undegraded":"2024-09-13T00:33:28.346342+0000","last_fullsized":"2024-09-13T00:33:28.346342+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:42:46.526231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352514+0000","last_change":"2024-09-13T00:32:23.133548+0000","last_active":"2024-09-13T00:33:28.352514+0000","last_peered":"2024-09-13T00:33:28.352514+0000","last_clean":"2024-09-13T00:33:28.352514+0000","last_became_active":"2024-09-13T00:32:23.132627+0000","last_became_peered":"2024-09-13T00:32:23.132627+0000","last_unstale":"2024-09-13T00:33:28.352514+0000","last_undegraded":"2024-09-13T00:33:28.352514+0000","last_fullsized":"2024-09-13T00:33:28.352514+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:46:28.698050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.353047+0000","last_change":"2024-09-13T00:32:23.173231+0000","last_active":"2024-09-13T00:33:28.353047+0000","last_peered":"2024-09-13T00:33:28.353047+0000","last_clean":"2024-09-13T00:33:28.353047+0000","last_became_active":"2024-09-13T00:32:23.172678+0000","last_became_peered":"2024-09-13T00:32:23.172678+0000","last_unstale":"2024-09-13T00:33:28.353047+0000","last_undegraded":"2024-09-13T00:33:28.353047+0000","last_fullsized":"2024-09-13T00:33:28.353047+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:07:25.182130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"51'79","reported_seq":56,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.790967+0000","last_change":"2024-09-13T00:28:31.831547+0000","last_active":"2024-09-13T00:33:28.790967+0000","last_peered":"2024-09-13T00:33:28.790967+0000","last_clean":"2024-09-13T00:33:28.790967+0000","last_became_active":"2024-09-13T00:28:31.819822+0000","last_became_peered":"2024-09-13T00:28:31.819822+0000","last_unstale":"2024-09-13T00:33:28.790967+0000","last_undegraded":"2024-09-13T00:33:28.790967+0000","last_fullsized":"2024-09-13T00:33:28.790967+0000","mapping_epoch":49,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":50,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:24:16.220413+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:24:16.220413+0000","last_clean_scrub_stamp":"2024-09-13T00:24:16.220413+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:19:22.270602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"55'1","reported_seq":27,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351748+0000","last_change":"2024-09-13T00:30:22.058593+0000","last_active":"2024-09-13T00:33:28.351748+0000","last_peered":"2024-09-13T00:33:28.351748+0000","last_clean":"2024-09-13T00:33:28.351748+0000","last_became_active":"2024-09-13T00:30:11.000634+0000","last_became_peered":"2024-09-13T00:30:11.000634+0000","last_unstale":"2024-09-13T00:33:28.351748+0000","last_undegraded":"2024-09-13T00:33:28.351748+0000","last_fullsized":"2024-09-13T00:33:28.351748+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:18:10.848845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096918099999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.353016+0000","last_change":"2024-09-13T00:32:23.118706+0000","last_active":"2024-09-13T00:33:28.353016+0000","last_peered":"2024-09-13T00:33:28.353016+0000","last_clean":"2024-09-13T00:33:28.353016+0000","last_became_active":"2024-09-13T00:32:23.117537+0000","last_became_peered":"2024-09-13T00:32:23.117537+0000","last_unstale":"2024-09-13T00:33:28.353016+0000","last_undegraded":"2024-09-13T00:33:28.353016+0000","last_fullsized":"2024-09-13T00:33:28.353016+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:40:21.484822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351986+0000","last_change":"2024-09-13T00:32:23.863503+0000","last_active":"2024-09-13T00:33:28.351986+0000","last_peered":"2024-09-13T00:33:28.351986+0000","last_clean":"2024-09-13T00:33:28.351986+0000","last_became_active":"2024-09-13T00:32:23.862635+0000","last_became_peered":"2024-09-13T00:32:23.862635+0000","last_unstale":"2024-09-13T00:33:28.351986+0000","last_undegraded":"2024-09-13T00:33:28.351986+0000","last_fullsized":"2024-09-13T00:33:28.351986+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:19:46.365911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.790736+0000","last_change":"2024-09-13T00:30:22.179791+0000","last_active":"2024-09-13T00:33:28.790736+0000","last_peered":"2024-09-13T00:33:28.790736+0000","last_clean":"2024-09-13T00:33:28.790736+0000","last_became_active":"2024-09-13T00:30:11.064343+0000","last_became_peered":"2024-09-13T00:30:11.064343+0000","last_unstale":"2024-09-13T00:33:28.790736+0000","last_undegraded":"2024-09-13T00:33:28.790736+0000","last_fullsized":"2024-09-13T00:33:28.790736+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:40:42.486916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062469399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.345404+0000","last_change":"2024-09-13T00:32:23.122555+0000","last_active":"2024-09-13T00:33:28.345404+0000","last_peered":"2024-09-13T00:33:28.345404+0000","last_clean":"2024-09-13T00:33:28.345404+0000","last_became_active":"2024-09-13T00:32:23.118952+0000","last_became_peered":"2024-09-13T00:32:23.118952+0000","last_unstale":"2024-09-13T00:33:28.345404+0000","last_undegraded":"2024-09-13T00:33:28.345404+0000","last_fullsized":"2024-09-13T00:33:28.345404+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:16:12.925852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.347155+0000","last_change":"2024-09-13T00:32:23.162688+0000","last_active":"2024-09-13T00:33:28.347155+0000","last_peered":"2024-09-13T00:33:28.347155+0000","last_clean":"2024-09-13T00:33:28.347155+0000","last_became_active":"2024-09-13T00:32:23.161593+0000","last_became_peered":"2024-09-13T00:32:23.161593+0000","last_unstale":"2024-09-13T00:33:28.347155+0000","last_undegraded":"2024-09-13T00:33:28.347155+0000","last_fullsized":"2024-09-13T00:33:28.347155+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:39:41.367544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.346694+0000","last_change":"2024-09-13T00:30:22.048961+0000","last_active":"2024-09-13T00:33:28.346694+0000","last_peered":"2024-09-13T00:33:28.346694+0000","last_clean":"2024-09-13T00:33:28.346694+0000","last_became_active":"2024-09-13T00:30:11.063178+0000","last_became_peered":"2024-09-13T00:30:11.063178+0000","last_unstale":"2024-09-13T00:33:28.346694+0000","last_undegraded":"2024-09-13T00:33:28.346694+0000","last_fullsized":"2024-09-13T00:33:28.346694+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:15:51.153544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067555600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351826+0000","last_change":"2024-09-13T00:32:23.108212+0000","last_active":"2024-09-13T00:33:28.351826+0000","last_peered":"2024-09-13T00:33:28.351826+0000","last_clean":"2024-09-13T00:33:28.351826+0000","last_became_active":"2024-09-13T00:32:23.103191+0000","last_became_peered":"2024-09-13T00:32:23.103191+0000","last_unstale":"2024-09-13T00:33:28.351826+0000","last_undegraded":"2024-09-13T00:33:28.351826+0000","last_fullsized":"2024-09-13T00:33:28.351826+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:50:25.094778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.353256+0000","last_change":"2024-09-13T00:32:23.160087+0000","last_active":"2024-09-13T00:33:28.353256+0000","last_peered":"2024-09-13T00:33:28.353256+0000","last_clean":"2024-09-13T00:33:28.353256+0000","last_became_active":"2024-09-13T00:32:23.159109+0000","last_became_peered":"2024-09-13T00:32:23.159109+0000","last_unstale":"2024-09-13T00:33:28.353256+0000","last_undegraded":"2024-09-13T00:33:28.353256+0000","last_fullsized":"2024-09-13T00:33:28.353256+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:13:08.870421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"57'2","reported_seq":28,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352114+0000","last_change":"2024-09-13T00:30:22.059692+0000","last_active":"2024-09-13T00:33:28.352114+0000","last_peered":"2024-09-13T00:33:28.352114+0000","last_clean":"2024-09-13T00:33:28.352114+0000","last_became_active":"2024-09-13T00:30:11.001662+0000","last_became_peered":"2024-09-13T00:30:11.001662+0000","last_unstale":"2024-09-13T00:33:28.352114+0000","last_undegraded":"2024-09-13T00:33:28.352114+0000","last_fullsized":"2024-09-13T00:33:28.352114+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:11:06.803316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002059707,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.347218+0000","last_change":"2024-09-13T00:32:23.118129+0000","last_active":"2024-09-13T00:33:28.347218+0000","last_peered":"2024-09-13T00:33:28.347218+0000","last_clean":"2024-09-13T00:33:28.347218+0000","last_became_active":"2024-09-13T00:32:23.116978+0000","last_became_peered":"2024-09-13T00:32:23.116978+0000","last_unstale":"2024-09-13T00:33:28.347218+0000","last_undegraded":"2024-09-13T00:33:28.347218+0000","last_fullsized":"2024-09-13T00:33:28.347218+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:08:03.526409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.353885+0000","last_change":"2024-09-13T00:32:23.172972+0000","last_active":"2024-09-13T00:33:28.353885+0000","last_peered":"2024-09-13T00:33:28.353885+0000","last_clean":"2024-09-13T00:33:28.353885+0000","last_became_active":"2024-09-13T00:32:23.172626+0000","last_became_peered":"2024-09-13T00:32:23.172626+0000","last_unstale":"2024-09-13T00:33:28.353885+0000","last_undegraded":"2024-09-13T00:33:28.353885+0000","last_fullsized":"2024-09-13T00:33:28.353885+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:11:25.868044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352536+0000","last_change":"2024-09-13T00:32:23.113059+0000","last_active":"2024-09-13T00:33:28.352536+0000","last_peered":"2024-09-13T00:33:28.352536+0000","last_clean":"2024-09-13T00:33:28.352536+0000","last_became_active":"2024-09-13T00:32:23.111858+0000","last_became_peered":"2024-09-13T00:32:23.111858+0000","last_unstale":"2024-09-13T00:33:28.352536+0000","last_undegraded":"2024-09-13T00:33:28.352536+0000","last_fullsized":"2024-09-13T00:33:28.352536+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T12:29:00.183110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.354590+0000","last_change":"2024-09-13T00:30:22.061674+0000","last_active":"2024-09-13T00:33:28.354590+0000","last_peered":"2024-09-13T00:33:28.354590+0000","last_clean":"2024-09-13T00:33:28.354590+0000","last_became_active":"2024-09-13T00:30:11.063404+0000","last_became_peered":"2024-09-13T00:30:11.063404+0000","last_unstale":"2024-09-13T00:33:28.354590+0000","last_undegraded":"2024-09-13T00:33:28.354590+0000","last_fullsized":"2024-09-13T00:33:28.354590+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:32:31.224759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053219400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.350987+0000","last_change":"2024-09-13T00:32:23.162665+0000","last_active":"2024-09-13T00:33:28.350987+0000","last_peered":"2024-09-13T00:33:28.350987+0000","last_clean":"2024-09-13T00:33:28.350987+0000","last_became_active":"2024-09-13T00:32:23.159005+0000","last_became_peered":"2024-09-13T00:32:23.159005+0000","last_unstale":"2024-09-13T00:33:28.350987+0000","last_undegraded":"2024-09-13T00:33:28.350987+0000","last_fullsized":"2024-09-13T00:33:28.350987+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:47:08.404863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.354318+0000","last_change":"2024-09-13T00:32:23.123503+0000","last_active":"2024-09-13T00:33:28.354318+0000","last_peered":"2024-09-13T00:33:28.354318+0000","last_clean":"2024-09-13T00:33:28.354318+0000","last_became_active":"2024-09-13T00:32:23.123111+0000","last_became_peered":"2024-09-13T00:32:23.123111+0000","last_unstale":"2024-09-13T00:33:28.354318+0000","last_undegraded":"2024-09-13T00:33:28.354318+0000","last_fullsized":"2024-09-13T00:33:28.354318+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T12:22:52.106632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.791421+0000","last_change":"2024-09-13T00:30:22.179990+0000","last_active":"2024-09-13T00:33:28.791421+0000","last_peered":"2024-09-13T00:33:28.791421+0000","last_clean":"2024-09-13T00:33:28.791421+0000","last_became_active":"2024-09-13T00:30:11.064377+0000","last_became_peered":"2024-09-13T00:30:11.064377+0000","last_unstale":"2024-09-13T00:33:28.791421+0000","last_undegraded":"2024-09-13T00:33:28.791421+0000","last_fullsized":"2024-09-13T00:33:28.791421+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:35:54.462078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077602300000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.346931+0000","last_change":"2024-09-13T00:32:23.860577+0000","last_active":"2024-09-13T00:33:28.346931+0000","last_peered":"2024-09-13T00:33:28.346931+0000","last_clean":"2024-09-13T00:33:28.346931+0000","last_became_active":"2024-09-13T00:32:23.859665+0000","last_became_peered":"2024-09-13T00:32:23.859665+0000","last_unstale":"2024-09-13T00:33:28.346931+0000","last_undegraded":"2024-09-13T00:33:28.346931+0000","last_fullsized":"2024-09-13T00:33:28.346931+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:28:41.285878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"62'2","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.355441+0000","last_change":"2024-09-13T00:32:23.114725+0000","last_active":"2024-09-13T00:33:28.355441+0000","last_peered":"2024-09-13T00:33:28.355441+0000","last_clean":"2024-09-13T00:33:28.355441+0000","last_became_active":"2024-09-13T00:32:23.113838+0000","last_became_peered":"2024-09-13T00:32:23.113838+0000","last_unstale":"2024-09-13T00:33:28.355441+0000","last_undegraded":"2024-09-13T00:33:28.355441+0000","last_fullsized":"2024-09-13T00:33:28.355441+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:06:02.251480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352554+0000","last_change":"2024-09-13T00:30:22.061997+0000","last_active":"2024-09-13T00:33:28.352554+0000","last_peered":"2024-09-13T00:33:28.352554+0000","last_clean":"2024-09-13T00:33:28.352554+0000","last_became_active":"2024-09-13T00:30:11.064525+0000","last_became_peered":"2024-09-13T00:30:11.064525+0000","last_unstale":"2024-09-13T00:33:28.352554+0000","last_undegraded":"2024-09-13T00:33:28.352554+0000","last_fullsized":"2024-09-13T00:33:28.352554+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:13:46.391207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085995799999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352319+0000","last_change":"2024-09-13T00:32:23.861978+0000","last_active":"2024-09-13T00:33:28.352319+0000","last_peered":"2024-09-13T00:33:28.352319+0000","last_clean":"2024-09-13T00:33:28.352319+0000","last_became_active":"2024-09-13T00:32:23.861171+0000","last_became_peered":"2024-09-13T00:32:23.861171+0000","last_unstale":"2024-09-13T00:33:28.352319+0000","last_undegraded":"2024-09-13T00:33:28.352319+0000","last_fullsized":"2024-09-13T00:33:28.352319+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:57:49.312865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.347363+0000","last_change":"2024-09-13T00:32:23.133518+0000","last_active":"2024-09-13T00:33:28.347363+0000","last_peered":"2024-09-13T00:33:28.347363+0000","last_clean":"2024-09-13T00:33:28.347363+0000","last_became_active":"2024-09-13T00:32:23.132162+0000","last_became_peered":"2024-09-13T00:32:23.132162+0000","last_unstale":"2024-09-13T00:33:28.347363+0000","last_undegraded":"2024-09-13T00:33:28.347363+0000","last_fullsized":"2024-09-13T00:33:28.347363+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:24:53.942970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351705+0000","last_change":"2024-09-13T00:30:22.062239+0000","last_active":"2024-09-13T00:33:28.351705+0000","last_peered":"2024-09-13T00:33:28.351705+0000","last_clean":"2024-09-13T00:33:28.351705+0000","last_became_active":"2024-09-13T00:30:11.000458+0000","last_became_peered":"2024-09-13T00:30:11.000458+0000","last_unstale":"2024-09-13T00:33:28.351705+0000","last_undegraded":"2024-09-13T00:33:28.351705+0000","last_fullsized":"2024-09-13T00:33:28.351705+0000","mapping_epoch":52,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":53,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:59:58.769064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00135603,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.345438+0000","last_change":"2024-09-13T00:32:23.151072+0000","last_active":"2024-09-13T00:33:28.345438+0000","last_peered":"2024-09-13T00:33:28.345438+0000","last_clean":"2024-09-13T00:33:28.345438+0000","last_became_active":"2024-09-13T00:32:23.150234+0000","last_became_peered":"2024-09-13T00:32:23.150234+0000","last_unstale":"2024-09-13T00:33:28.345438+0000","last_undegraded":"2024-09-13T00:33:28.345438+0000","last_fullsized":"2024-09-13T00:33:28.345438+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:03:02.780398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.347105+0000","last_change":"2024-09-13T00:32:23.160485+0000","last_active":"2024-09-13T00:33:28.347105+0000","last_peered":"2024-09-13T00:33:28.347105+0000","last_clean":"2024-09-13T00:33:28.347105+0000","last_became_active":"2024-09-13T00:32:23.158982+0000","last_became_peered":"2024-09-13T00:32:23.158982+0000","last_unstale":"2024-09-13T00:33:28.347105+0000","last_undegraded":"2024-09-13T00:33:28.347105+0000","last_fullsized":"2024-09-13T00:33:28.347105+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:54:03.153668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"63'7","reported_seq":26,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.345974+0000","last_change":"2024-09-13T00:32:23.100206+0000","last_active":"2024-09-13T00:33:28.345974+0000","last_peered":"2024-09-13T00:33:28.345974+0000","last_clean":"2024-09-13T00:33:28.345974+0000","last_became_active":"2024-09-13T00:32:23.098540+0000","last_became_peered":"2024-09-13T00:32:23.098540+0000","last_unstale":"2024-09-13T00:33:28.345974+0000","last_undegraded":"2024-09-13T00:33:28.345974+0000","last_fullsized":"2024-09-13T00:33:28.345974+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:20:56.437622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351075+0000","last_change":"2024-09-13T00:32:23.164292+0000","last_active":"2024-09-13T00:33:28.351075+0000","last_peered":"2024-09-13T00:33:28.351075+0000","last_clean":"2024-09-13T00:33:28.351075+0000","last_became_active":"2024-09-13T00:32:23.160050+0000","last_became_peered":"2024-09-13T00:32:23.160050+0000","last_unstale":"2024-09-13T00:33:28.351075+0000","last_undegraded":"2024-09-13T00:33:28.351075+0000","last_fullsized":"2024-09-13T00:33:28.351075+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:45:02.738945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"62'3","reported_seq":19,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.791297+0000","last_change":"2024-09-13T00:32:23.118269+0000","last_active":"2024-09-13T00:33:28.791297+0000","last_peered":"2024-09-13T00:33:28.791297+0000","last_clean":"2024-09-13T00:33:28.791297+0000","last_became_active":"2024-09-13T00:32:23.117466+0000","last_became_peered":"2024-09-13T00:32:23.117466+0000","last_unstale":"2024-09-13T00:33:28.791297+0000","last_undegraded":"2024-09-13T00:33:28.791297+0000","last_fullsized":"2024-09-13T00:33:28.791297+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:29:52.479917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351740+0000","last_change":"2024-09-13T00:32:23.126158+0000","last_active":"2024-09-13T00:33:28.351740+0000","last_peered":"2024-09-13T00:33:28.351740+0000","last_clean":"2024-09-13T00:33:28.351740+0000","last_became_active":"2024-09-13T00:32:23.125344+0000","last_became_peered":"2024-09-13T00:32:23.125344+0000","last_unstale":"2024-09-13T00:33:28.351740+0000","last_undegraded":"2024-09-13T00:33:28.351740+0000","last_fullsized":"2024-09-13T00:33:28.351740+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:06:33.501725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.353442+0000","last_change":"2024-09-13T00:32:23.864198+0000","last_active":"2024-09-13T00:33:28.353442+0000","last_peered":"2024-09-13T00:33:28.353442+0000","last_clean":"2024-09-13T00:33:28.353442+0000","last_became_active":"2024-09-13T00:32:23.863396+0000","last_became_peered":"2024-09-13T00:32:23.863396+0000","last_unstale":"2024-09-13T00:33:28.353442+0000","last_undegraded":"2024-09-13T00:33:28.353442+0000","last_fullsized":"2024-09-13T00:33:28.353442+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:05:47.048726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.346812+0000","last_change":"2024-09-13T00:32:23.129859+0000","last_active":"2024-09-13T00:33:28.346812+0000","last_peered":"2024-09-13T00:33:28.346812+0000","last_clean":"2024-09-13T00:33:28.346812+0000","last_became_active":"2024-09-13T00:32:23.129153+0000","last_became_peered":"2024-09-13T00:32:23.129153+0000","last_unstale":"2024-09-13T00:33:28.346812+0000","last_undegraded":"2024-09-13T00:33:28.346812+0000","last_fullsized":"2024-09-13T00:33:28.346812+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:07:04.367114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352236+0000","last_change":"2024-09-13T00:32:23.174513+0000","last_active":"2024-09-13T00:33:28.352236+0000","last_peered":"2024-09-13T00:33:28.352236+0000","last_clean":"2024-09-13T00:33:28.352236+0000","last_became_active":"2024-09-13T00:32:23.173751+0000","last_became_peered":"2024-09-13T00:32:23.173751+0000","last_unstale":"2024-09-13T00:33:28.352236+0000","last_undegraded":"2024-09-13T00:33:28.352236+0000","last_fullsized":"2024-09-13T00:33:28.352236+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:07:14.494788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351498+0000","last_change":"2024-09-13T00:32:23.162492+0000","last_active":"2024-09-13T00:33:28.351498+0000","last_peered":"2024-09-13T00:33:28.351498+0000","last_clean":"2024-09-13T00:33:28.351498+0000","last_became_active":"2024-09-13T00:32:23.162061+0000","last_became_peered":"2024-09-13T00:32:23.162061+0000","last_unstale":"2024-09-13T00:33:28.351498+0000","last_undegraded":"2024-09-13T00:33:28.351498+0000","last_fullsized":"2024-09-13T00:33:28.351498+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:49:36.897842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352410+0000","last_change":"2024-09-13T00:32:23.135038+0000","last_active":"2024-09-13T00:33:28.352410+0000","last_peered":"2024-09-13T00:33:28.352410+0000","last_clean":"2024-09-13T00:33:28.352410+0000","last_became_active":"2024-09-13T00:32:23.134333+0000","last_became_peered":"2024-09-13T00:32:23.134333+0000","last_unstale":"2024-09-13T00:33:28.352410+0000","last_undegraded":"2024-09-13T00:33:28.352410+0000","last_fullsized":"2024-09-13T00:33:28.352410+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:53:08.926676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.347022+0000","last_change":"2024-09-13T00:32:23.861211+0000","last_active":"2024-09-13T00:33:28.347022+0000","last_peered":"2024-09-13T00:33:28.347022+0000","last_clean":"2024-09-13T00:33:28.347022+0000","last_became_active":"2024-09-13T00:32:23.860309+0000","last_became_peered":"2024-09-13T00:32:23.860309+0000","last_unstale":"2024-09-13T00:33:28.347022+0000","last_undegraded":"2024-09-13T00:33:28.347022+0000","last_fullsized":"2024-09-13T00:33:28.347022+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:54:28.941726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351199+0000","last_change":"2024-09-13T00:32:23.131011+0000","last_active":"2024-09-13T00:33:28.351199+0000","last_peered":"2024-09-13T00:33:28.351199+0000","last_clean":"2024-09-13T00:33:28.351199+0000","last_became_active":"2024-09-13T00:32:23.130169+0000","last_became_peered":"2024-09-13T00:32:23.130169+0000","last_unstale":"2024-09-13T00:33:28.351199+0000","last_undegraded":"2024-09-13T00:33:28.351199+0000","last_fullsized":"2024-09-13T00:33:28.351199+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:28:55.512823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.355216+0000","last_change":"2024-09-13T00:32:23.172802+0000","last_active":"2024-09-13T00:33:28.355216+0000","last_peered":"2024-09-13T00:33:28.355216+0000","last_clean":"2024-09-13T00:33:28.355216+0000","last_became_active":"2024-09-13T00:32:23.172481+0000","last_became_peered":"2024-09-13T00:32:23.172481+0000","last_unstale":"2024-09-13T00:33:28.355216+0000","last_undegraded":"2024-09-13T00:33:28.355216+0000","last_fullsized":"2024-09-13T00:33:28.355216+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:59:15.071966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'2","reported_seq":16,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.345334+0000","last_change":"2024-09-13T00:32:23.102122+0000","last_active":"2024-09-13T00:33:28.345334+0000","last_peered":"2024-09-13T00:33:28.345334+0000","last_clean":"2024-09-13T00:33:28.345334+0000","last_became_active":"2024-09-13T00:32:23.098269+0000","last_became_peered":"2024-09-13T00:32:23.098269+0000","last_unstale":"2024-09-13T00:33:28.345334+0000","last_undegraded":"2024-09-13T00:33:28.345334+0000","last_fullsized":"2024-09-13T00:33:28.345334+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:33:52.642515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.346577+0000","last_change":"2024-09-13T00:32:23.165471+0000","last_active":"2024-09-13T00:33:28.346577+0000","last_peered":"2024-09-13T00:33:28.346577+0000","last_clean":"2024-09-13T00:33:28.346577+0000","last_became_active":"2024-09-13T00:32:23.164918+0000","last_became_peered":"2024-09-13T00:32:23.164918+0000","last_unstale":"2024-09-13T00:33:28.346577+0000","last_undegraded":"2024-09-13T00:33:28.346577+0000","last_fullsized":"2024-09-13T00:33:28.346577+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:20:24.972830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'2","reported_seq":16,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.347018+0000","last_change":"2024-09-13T00:32:23.129053+0000","last_active":"2024-09-13T00:33:28.347018+0000","last_peered":"2024-09-13T00:33:28.347018+0000","last_clean":"2024-09-13T00:33:28.347018+0000","last_became_active":"2024-09-13T00:32:23.128636+0000","last_became_peered":"2024-09-13T00:32:23.128636+0000","last_unstale":"2024-09-13T00:33:28.347018+0000","last_undegraded":"2024-09-13T00:33:28.347018+0000","last_fullsized":"2024-09-13T00:33:28.347018+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:09:01.573929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.346971+0000","last_change":"2024-09-13T00:32:23.167820+0000","last_active":"2024-09-13T00:33:28.346971+0000","last_peered":"2024-09-13T00:33:28.346971+0000","last_clean":"2024-09-13T00:33:28.346971+0000","last_became_active":"2024-09-13T00:32:23.166972+0000","last_became_peered":"2024-09-13T00:32:23.166972+0000","last_unstale":"2024-09-13T00:33:28.346971+0000","last_undegraded":"2024-09-13T00:33:28.346971+0000","last_fullsized":"2024-09-13T00:33:28.346971+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:21:47.508081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"62'2","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.351911+0000","last_change":"2024-09-13T00:32:23.124131+0000","last_active":"2024-09-13T00:33:28.351911+0000","last_peered":"2024-09-13T00:33:28.351911+0000","last_clean":"2024-09-13T00:33:28.351911+0000","last_became_active":"2024-09-13T00:32:23.123871+0000","last_became_peered":"2024-09-13T00:32:23.123871+0000","last_unstale":"2024-09-13T00:33:28.351911+0000","last_undegraded":"2024-09-13T00:33:28.351911+0000","last_fullsized":"2024-09-13T00:33:28.351911+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:56:02.460621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.345202+0000","last_change":"2024-09-13T00:32:23.167683+0000","last_active":"2024-09-13T00:33:28.345202+0000","last_peered":"2024-09-13T00:33:28.345202+0000","last_clean":"2024-09-13T00:33:28.345202+0000","last_became_active":"2024-09-13T00:32:23.166379+0000","last_became_peered":"2024-09-13T00:32:23.166379+0000","last_unstale":"2024-09-13T00:33:28.345202+0000","last_undegraded":"2024-09-13T00:33:28.345202+0000","last_fullsized":"2024-09-13T00:33:28.345202+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:56:53.232461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"62'2","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.352412+0000","last_change":"2024-09-13T00:32:23.118163+0000","last_active":"2024-09-13T00:33:28.352412+0000","last_peered":"2024-09-13T00:33:28.352412+0000","last_clean":"2024-09-13T00:33:28.352412+0000","last_became_active":"2024-09-13T00:32:23.117401+0000","last_became_peered":"2024-09-13T00:32:23.117401+0000","last_unstale":"2024-09-13T00:33:28.352412+0000","last_undegraded":"2024-09-13T00:33:28.352412+0000","last_fullsized":"2024-09-13T00:33:28.352412+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:17:55.385242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.354867+0000","last_change":"2024-09-13T00:32:23.166559+0000","last_active":"2024-09-13T00:33:28.354867+0000","last_peered":"2024-09-13T00:33:28.354867+0000","last_clean":"2024-09-13T00:33:28.354867+0000","last_became_active":"2024-09-13T00:32:23.165591+0000","last_became_peered":"2024-09-13T00:32:23.165591+0000","last_unstale":"2024-09-13T00:33:28.354867+0000","last_undegraded":"2024-09-13T00:33:28.354867+0000","last_fullsized":"2024-09-13T00:33:28.354867+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:21:34.838584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'2","reported_seq":16,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.355333+0000","last_change":"2024-09-13T00:32:23.117026+0000","last_active":"2024-09-13T00:33:28.355333+0000","last_peered":"2024-09-13T00:33:28.355333+0000","last_clean":"2024-09-13T00:33:28.355333+0000","last_became_active":"2024-09-13T00:32:23.116372+0000","last_became_peered":"2024-09-13T00:32:23.116372+0000","last_unstale":"2024-09-13T00:33:28.355333+0000","last_undegraded":"2024-09-13T00:33:28.355333+0000","last_fullsized":"2024-09-13T00:33:28.355333+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:02:40.393662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":65,"state":"active+clean","last_fresh":"2024-09-13T00:33:29.815946+0000","last_change":"2024-09-13T00:33:29.815946+0000","last_active":"2024-09-13T00:33:29.815946+0000","last_peered":"2024-09-13T00:33:29.815946+0000","last_clean":"2024-09-13T00:33:29.815946+0000","last_became_active":"2024-09-13T00:33:29.815260+0000","last_became_peered":"2024-09-13T00:33:29.815260+0000","last_unstale":"2024-09-13T00:33:29.815946+0000","last_undegraded":"2024-09-13T00:33:29.815946+0000","last_fullsized":"2024-09-13T00:33:29.815946+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:05:55.292319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'5","reported_seq":21,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-09-13T00:33:28.346768+0000","last_change":"2024-09-13T00:32:23.127589+0000","last_active":"2024-09-13T00:33:28.346768+0000","last_peered":"2024-09-13T00:33:28.346768+0000","last_clean":"2024-09-13T00:33:28.346768+0000","last_became_active":"2024-09-13T00:32:23.126425+0000","last_became_peered":"2024-09-13T00:32:23.126425+0000","last_unstale":"2024-09-13T00:33:28.346768+0000","last_undegraded":"2024-09-13T00:33:28.346768+0000","last_fullsized":"2024-09-13T00:33:28.346768+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:32:44.249447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":47,"num_read_kb":49,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":126,"num_read_kb":231,"num_write":113,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430281,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298616,"kb_used_data":1256,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446536,"statfs":{"total":95995035648,"available":95689252864,"internally_reserved":0,"allocated":1286144,"data_stored":886168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.89000000000000001,"15min":0.85999999999999999},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.1839999999999999,"5min":1.345,"15min":1.345},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.91200000000000003,"15min":0.874},"min":{"1min":0.67800000000000005,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.179,"5min":1.53,"15min":1.53},"last":1.163}]},{"osd":1,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.84299999999999997,"15min":0.82499999999999996},"min":{"1min":0.54400000000000004,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.224,"5min":1.319,"15min":1.319},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.92700000000000005,"15min":0.86099999999999999},"min":{"1min":0.61099999999999999,"5min":0.435,"15min":0.435},"max":{"1min":1.1910000000000001,"5min":1.385,"15min":1.385},"last":1.024}]},{"osd":2,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.93000000000000005,"15min":0.92100000000000004},"min":{"1min":0.70599999999999996,"5min":0.434,"15min":0.434},"max":{"1min":1.6100000000000001,"5min":1.6100000000000001,"15min":1.6100000000000001},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.91700000000000004,"15min":0.89700000000000002},"min":{"1min":0.57199999999999995,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.389,"5min":1.389,"15min":1.389},"last":0.86799999999999999}]},{"osd":3,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":0.94599999999999995,"15min":0.89100000000000001},"min":{"1min":0.64000000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":1.736,"5min":1.736,"15min":1.736},"last":0.92400000000000004},{"interface":"front","average":{"1min":0.91900000000000004,"5min":0.93899999999999995,"15min":0.93899999999999995},"min":{"1min":0.64500000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.177,"5min":1.306,"15min":1.306},"last":0.95399999999999996}]},{"osd":4,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.89000000000000001,"15min":0.89600000000000002},"min":{"1min":0.628,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.3129999999999999,"5min":1.4790000000000001,"15min":1.4790000000000001},"last":1.2230000000000001},{"interface":"front","average":{"1min":1.018,"5min":0.97899999999999998,"15min":0.90800000000000003},"min":{"1min":0.67500000000000004,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.5109999999999999,"5min":1.6160000000000001,"15min":1.6160000000000001},"last":0.995}]},{"osd":5,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.90300000000000002,"15min":0.88600000000000001},"min":{"1min":0.72899999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.1200000000000001,"5min":1.506,"15min":1.506},"last":1.131},{"interface":"front","average":{"1min":1.02,"5min":0.98099999999999998,"15min":0.94399999999999995},"min":{"1min":0.72299999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.2829999999999999,"5min":1.5800000000000001,"15min":1.5800000000000001},"last":1.2649999999999999}]},{"osd":6,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.0129999999999999,"15min":0.97699999999999998},"min":{"1min":0.85999999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":1.343,"5min":1.5149999999999999,"15min":1.5149999999999999},"last":1.3049999999999999},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.0640000000000001,"15min":1.036},"min":{"1min":0.76600000000000001,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.8140000000000001,"5min":1.8140000000000001,"15min":1.8140000000000001},"last":1.0820000000000001}]}]},{"osd":6,"up_from":42,"seq":180388626515,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298596,"kb_used_data":1240,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446556,"statfs":{"total":95995035648,"available":95689273344,"internally_reserved":0,"allocated":1269760,"data_stored":885482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.88200000000000001,"15min":0.84199999999999997},"min":{"1min":0.51300000000000001,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":2.5659999999999998,"5min":2.5659999999999998,"15min":2.5659999999999998},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.03,"5min":0.92200000000000004,"15min":0.873},"min":{"1min":0.65300000000000002,"5min":0.56000000000000005,"15min":0.56000000000000005},"max":{"1min":2.5310000000000001,"5min":2.5310000000000001,"15min":2.5310000000000001},"last":0.59099999999999997}]},{"osd":1,"last update":"Fri Sep 13 00:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.89700000000000002,"15min":0.91900000000000004},"min":{"1min":0.69799999999999995,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":2.5920000000000001,"5min":2.5920000000000001,"15min":2.5920000000000001},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.90100000000000002,"15min":0.89300000000000002},"min":{"1min":0.70399999999999996,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.2689999999999999,"5min":1.2689999999999999,"15min":1.2689999999999999},"last":0.98599999999999999}]},{"osd":2,"last update":"Fri Sep 13 00:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.95899999999999996,"15min":0.91600000000000004},"min":{"1min":0.60299999999999998,"5min":0.443,"15min":0.443},"max":{"1min":1.24,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.2470000000000001},{"interface":"front","average":{"1min":1.032,"5min":0.91100000000000003,"15min":0.91800000000000004},"min":{"1min":0.71699999999999997,"5min":0.498,"15min":0.498},"max":{"1min":2.4889999999999999,"5min":2.4889999999999999,"15min":2.4889999999999999},"last":1.0629999999999999}]},{"osd":3,"last update":"Fri Sep 13 00:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.97399999999999998,"5min":0.90600000000000003,"15min":0.91400000000000003},"min":{"1min":0.82799999999999996,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":0.85299999999999998},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.0069999999999999,"15min":0.95899999999999996},"min":{"1min":0.73399999999999999,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.4339999999999999,"5min":1.4339999999999999,"15min":1.4339999999999999},"last":1.3779999999999999}]},{"osd":4,"last update":"Fri Sep 13 00:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":0.85099999999999998,"15min":0.83299999999999996},"min":{"1min":0.63600000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.121,"5min":1.1220000000000001,"15min":1.1220000000000001},"last":1.1619999999999999},{"interface":"front","average":{"1min":0.95799999999999996,"5min":0.94899999999999995,"15min":0.89100000000000001},"min":{"1min":0.69299999999999995,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.3169999999999999,"5min":1.403,"15min":1.403},"last":1.099}]},{"osd":5,"last update":"Fri Sep 13 00:33:54 2024","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.98199999999999998,"15min":0.874},"min":{"1min":0.69199999999999995,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.3540000000000001,"5min":1.357,"15min":1.357},"last":1.3320000000000001},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.97399999999999998,"15min":0.94199999999999995},"min":{"1min":0.84899999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.179,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":1.212}]},{"osd":7,"last update":"Fri Sep 13 00:33:42 2024","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.0249999999999999,"15min":0.96899999999999997},"min":{"1min":0.80900000000000005,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.254,"5min":1.3959999999999999,"15min":1.3959999999999999},"last":1.425},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.1359999999999999,"15min":1.0740000000000001},"min":{"1min":0.995,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.498,"5min":1.498,"15min":1.498},"last":1.4650000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822749,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":667648,"data_stored":294132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:34:09 2024","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.86699999999999999,"15min":0.82999999999999996},"min":{"1min":0.61299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.129,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":1.0149999999999999},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.93000000000000005,"15min":0.86099999999999999},"min":{"1min":0.61099999999999999,"5min":0.51600000000000001,"15min":0.51600000000000001},"max":{"1min":1.1990000000000001,"5min":1.2909999999999999,"15min":1.3120000000000001},"last":1.202}]},{"osd":1,"last update":"Fri Sep 13 00:34:09 2024","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.92700000000000005,"15min":0.85599999999999998},"min":{"1min":0.70699999999999996,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.3680000000000001,"5min":1.3680000000000001,"15min":1.5109999999999999},"last":1.048},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.90200000000000002,"15min":0.86199999999999999},"min":{"1min":0.66100000000000003,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.214,"5min":1.516,"15min":1.516},"last":0.82199999999999995}]},{"osd":2,"last update":"Fri Sep 13 00:34:09 2024","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":0.85499999999999998,"15min":0.84299999999999997},"min":{"1min":0.67400000000000004,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.0700000000000001,"5min":1.306,"15min":1.306},"last":1.173},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.96899999999999997,"15min":0.876},"min":{"1min":0.79600000000000004,"5min":0.62,"15min":0.53500000000000003},"max":{"1min":1.149,"5min":3.556,"15min":3.556},"last":0.86599999999999999}]},{"osd":3,"last update":"Fri Sep 13 00:34:09 2024","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":0.92400000000000004,"15min":0.876},"min":{"1min":0.70599999999999996,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.1519999999999999,"5min":1.3700000000000001,"15min":1.3700000000000001},"last":1.099},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.97699999999999998,"15min":0.88200000000000001},"min":{"1min":0.78600000000000003,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.3220000000000001,"5min":1.456,"15min":1.456},"last":0.93000000000000005}]},{"osd":4,"last update":"Fri Sep 13 00:34:09 2024","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.84599999999999997,"15min":0.71499999999999997},"min":{"1min":0.66700000000000004,"5min":0.376,"15min":0.34499999999999997},"max":{"1min":1.262,"5min":1.262,"15min":1.2829999999999999},"last":0.755},{"interface":"front","average":{"1min":0.875,"5min":0.92900000000000005,"15min":0.76000000000000001},"min":{"1min":0.58999999999999997,"5min":0.46400000000000002,"15min":0.30599999999999999},"max":{"1min":1.1419999999999999,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":0.88400000000000001}]},{"osd":6,"last update":"Fri Sep 13 00:33:51 2024","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.93700000000000006,"15min":0.871},"min":{"1min":0.65200000000000002,"5min":0.44600000000000001,"15min":0.41399999999999998},"max":{"1min":1.206,"5min":1.3280000000000001,"15min":1.3280000000000001},"last":0.98899999999999999},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.073,"15min":0.96699999999999997},"min":{"1min":0.72799999999999998,"5min":0.45700000000000002,"15min":0.45300000000000001},"max":{"1min":1.2889999999999999,"5min":1.603,"15min":1.603},"last":1.1319999999999999}]},{"osd":7,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.1000000000000001,"15min":1.0880000000000001},"min":{"1min":0.81100000000000005,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.3560000000000001,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":1.2509999999999999},{"interface":"front","average":{"1min":1.0349999999999999,"5min":0.995,"15min":1.0229999999999999},"min":{"1min":0.73199999999999998,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.248,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.95799999999999996}]}]},{"osd":4,"up_from":30,"seq":128849018982,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298032,"kb_used_data":672,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447120,"statfs":{"total":95995035648,"available":95689850880,"internally_reserved":0,"allocated":688128,"data_stored":295730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:34:17 2024","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.89700000000000002,"15min":0.81399999999999995},"min":{"1min":0.49199999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.1830000000000001,"5min":1.216,"15min":1.216},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.95399999999999996,"5min":0.91800000000000004,"15min":0.84299999999999997},"min":{"1min":0.624,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.2470000000000001,"5min":1.284,"15min":1.284},"last":1.169}]},{"osd":1,"last update":"Fri Sep 13 00:34:17 2024","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.90500000000000003,"15min":0.83499999999999996},"min":{"1min":0.55700000000000005,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.1180000000000001,"5min":1.2529999999999999,"15min":1.405},"last":1.089},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.93600000000000005,"15min":0.86299999999999999},"min":{"1min":0.55900000000000005,"5min":0.49299999999999999,"15min":0.438},"max":{"1min":1.2949999999999999,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.80300000000000005}]},{"osd":2,"last update":"Fri Sep 13 00:34:17 2024","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.92500000000000004,"15min":0.85399999999999998},"min":{"1min":0.72799999999999998,"5min":0.54600000000000004,"15min":0.51100000000000001},"max":{"1min":1.1539999999999999,"5min":1.196,"15min":1.196},"last":1.006},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.91000000000000003,"15min":0.85499999999999998},"min":{"1min":0.627,"5min":0.52600000000000002,"15min":0.48399999999999999},"max":{"1min":1.2629999999999999,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":0.96499999999999997}]},{"osd":3,"last update":"Fri Sep 13 00:34:17 2024","interfaces":[{"interface":"back","average":{"1min":0.93500000000000005,"5min":0.93400000000000005,"15min":0.90300000000000002},"min":{"1min":0.72299999999999998,"5min":0.53500000000000003,"15min":0.46600000000000003},"max":{"1min":1.1819999999999999,"5min":1.575,"15min":1.587},"last":1.119},{"interface":"front","average":{"1min":0.98499999999999999,"5min":0.95599999999999996,"15min":0.90000000000000002},"min":{"1min":0.72599999999999998,"5min":0.437,"15min":0.437},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":0.89400000000000002}]},{"osd":5,"last update":"Fri Sep 13 00:34:07 2024","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.92100000000000004,"15min":0.73699999999999999},"min":{"1min":0.63500000000000001,"5min":0.315,"15min":0.315},"max":{"1min":1.149,"5min":1.405,"15min":1.405},"last":0.91400000000000003},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.93000000000000005,"15min":0.76500000000000001},"min":{"1min":0.68300000000000005,"5min":0.35199999999999998,"15min":0.33100000000000002},"max":{"1min":1.1499999999999999,"5min":1.472,"15min":1.472},"last":0.76400000000000001}]},{"osd":6,"last update":"Fri Sep 13 00:33:48 2024","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":0.95699999999999996,"15min":0.84799999999999998},"min":{"1min":0.64000000000000001,"5min":0.39100000000000001,"15min":0.34499999999999997},"max":{"1min":1.2430000000000001,"5min":1.347,"15min":1.347},"last":1.042},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.016,"15min":0.91200000000000003},"min":{"1min":0.82099999999999995,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.294,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":1.2330000000000001}]},{"osd":7,"last update":"Fri Sep 13 00:33:39 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.999,"15min":1.0209999999999999},"min":{"1min":0.69599999999999995,"5min":0.37,"15min":0.37},"max":{"1min":1.216,"5min":1.8480000000000001,"15min":1.8480000000000001},"last":1.0580000000000001},{"interface":"front","average":{"1min":1.121,"5min":1.0429999999999999,"15min":1.026},"min":{"1min":0.749,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":1.395,"5min":1.464,"15min":1.464},"last":1.1479999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182512,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298008,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447144,"statfs":{"total":95995035648,"available":95689875456,"internally_reserved":0,"allocated":667648,"data_stored":294122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:33:25 2024","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.66200000000000003,"15min":0.64700000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.91700000000000004,"5min":1.175,"15min":1.175},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.748,"5min":0.72699999999999998,"15min":0.747},"min":{"1min":0.32500000000000001,"5min":0.307,"15min":0.307},"max":{"1min":1.0669999999999999,"5min":1.1619999999999999,"15min":2.1400000000000001},"last":0.74299999999999999}]},{"osd":1,"last update":"Fri Sep 13 00:33:25 2024","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":0.755,"15min":0.76400000000000001},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.0589999999999999,"5min":1.1319999999999999,"15min":2.2029999999999998},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.68899999999999995,"5min":0.72099999999999997,"15min":0.71099999999999997},"min":{"1min":0.34399999999999997,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.0289999999999999,"5min":1.274,"15min":1.274},"last":0.78100000000000003}]},{"osd":2,"last update":"Fri Sep 13 00:33:25 2024","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.77600000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":1.4299999999999999,"5min":1.4299999999999999,"15min":1.4299999999999999},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.75700000000000001,"15min":0.72099999999999997},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.212,"5min":1.212,"15min":1.212},"last":0.69499999999999995}]},{"osd":4,"last update":"Fri Sep 13 00:33:18 2024","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.91100000000000003,"15min":0.88800000000000001},"min":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.1850000000000001,"5min":1.2410000000000001,"15min":1.2410000000000001},"last":1.1339999999999999},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.98699999999999999,"15min":0.91500000000000004},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.2909999999999999,"5min":1.3120000000000001,"15min":1.3120000000000001},"last":1.01}]},{"osd":5,"last update":"Fri Sep 13 00:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.97599999999999998,"15min":0.95799999999999996},"min":{"1min":0.63600000000000001,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.2909999999999999,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.98399999999999999,"5min":0.97499999999999998,"15min":0.95199999999999996},"min":{"1min":0.67500000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.472,"5min":1.732,"15min":1.732},"last":1.1060000000000001}]},{"osd":6,"last update":"Fri Sep 13 00:33:51 2024","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.022,"15min":1.0840000000000001},"min":{"1min":0.66100000000000003,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.26,"5min":1.4570000000000001,"15min":2.6040000000000001},"last":1.071},{"interface":"front","average":{"1min":0.999,"5min":1.014,"15min":1.03},"min":{"1min":0.61099999999999999,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":1.302,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":1.228}]},{"osd":7,"last update":"Fri Sep 13 00:33:41 2024","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.04,"15min":1.097},"min":{"1min":0.68899999999999995,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.417,"5min":1.619,"15min":1.619},"last":1.1830000000000001},{"interface":"front","average":{"1min":1.024,"5min":1.018,"15min":1.0740000000000001},"min":{"1min":0.623,"5min":0.53800000000000003,"15min":0.53800000000000003},"max":{"1min":1.3799999999999999,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":1.042}]}]},{"osd":2,"up_from":18,"seq":77309411452,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298012,"kb_used_data":656,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447140,"statfs":{"total":95995035648,"available":95689871360,"internally_reserved":0,"allocated":671744,"data_stored":294636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:33:32 2024","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":0.67300000000000004,"15min":0.58999999999999997},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":1.125,"5min":1.125,"15min":1.125},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.67700000000000005,"15min":0.59899999999999998},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.98099999999999998,"5min":1.113,"15min":1.113},"last":0.73099999999999998}]},{"osd":1,"last update":"Fri Sep 13 00:33:32 2024","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":0.73899999999999999,"15min":0.66200000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33800000000000002},"max":{"1min":1.1990000000000001,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.755,"15min":0.65000000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":1.101,"5min":1.8680000000000001,"15min":1.8680000000000001},"last":0.78100000000000003}]},{"osd":3,"last update":"Fri Sep 13 00:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.83599999999999997,"5min":0.82099999999999995,"15min":0.77000000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.2110000000000001,"5min":1.3580000000000001,"15min":1.3580000000000001},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.79200000000000004,"5min":0.82099999999999995,"15min":0.80300000000000005},"min":{"1min":0.38300000000000001,"5min":0.377,"15min":0.377},"max":{"1min":1.2909999999999999,"5min":1.3049999999999999,"15min":1.319},"last":1}]},{"osd":4,"last update":"Fri Sep 13 00:33:24 2024","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.98399999999999999,"15min":0.93300000000000005},"min":{"1min":0.71699999999999997,"5min":0.60299999999999998,"15min":0.60299999999999998},"max":{"1min":1.25,"5min":1.395,"15min":1.395},"last":1.0609999999999999},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.0409999999999999,"15min":0.94599999999999995},"min":{"1min":0.73799999999999999,"5min":0.71599999999999997,"15min":0.71599999999999997},"max":{"1min":1.3129999999999999,"5min":1.6599999999999999,"15min":1.6599999999999999},"last":1.0349999999999999}]},{"osd":5,"last update":"Fri Sep 13 00:34:12 2024","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.97399999999999998,"15min":0.95799999999999996},"min":{"1min":0.70099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"max":{"1min":1.244,"5min":1.5900000000000001,"15min":1.5900000000000001},"last":1.0800000000000001},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.95099999999999996,"15min":0.91500000000000004},"min":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"max":{"1min":1.1990000000000001,"5min":1.5509999999999999,"15min":1.5509999999999999},"last":0.874}]},{"osd":6,"last update":"Fri Sep 13 00:33:53 2024","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":0.97899999999999998,"15min":0.98599999999999999},"min":{"1min":0.63500000000000001,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.2589999999999999,"5min":1.641,"15min":1.641},"last":0.96499999999999997},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.0289999999999999,"15min":1.0249999999999999},"min":{"1min":0.752,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.163,"5min":1.51,"15min":1.51},"last":1.1699999999999999}]},{"osd":7,"last update":"Fri Sep 13 00:33:36 2024","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.0640000000000001,"15min":1.0780000000000001},"min":{"1min":0.81699999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"max":{"1min":1.3759999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"last":1.1279999999999999},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.0509999999999999,"15min":1.0800000000000001},"min":{"1min":0.56699999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.4730000000000001,"5min":1.6220000000000001,"15min":1.6220000000000001},"last":0.90200000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574984,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298012,"kb_used_data":652,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447140,"statfs":{"total":95995035648,"available":95689871360,"internally_reserved":0,"allocated":667648,"data_stored":294051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:33:51 2024","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.63800000000000001,"15min":0.56100000000000005},"min":{"1min":0.307,"5min":0.307,"15min":0.25900000000000001},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.497},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.76700000000000002,"15min":0.64800000000000002},"min":{"1min":0.47799999999999998,"5min":0.30099999999999999,"15min":0.28299999999999997},"max":{"1min":1.7170000000000001,"5min":1.7170000000000001,"15min":1.8879999999999999},"last":0.74199999999999999}]},{"osd":2,"last update":"Fri Sep 13 00:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.81000000000000005,"5min":0.75600000000000001,"15min":0.64400000000000002},"min":{"1min":0.45400000000000001,"5min":0.36199999999999999,"15min":0.28699999999999998},"max":{"1min":1.825,"5min":1.825,"15min":1.825},"last":0.95299999999999996},{"interface":"front","average":{"1min":0.75,"5min":0.70599999999999996,"15min":0.63},"min":{"1min":0.51700000000000002,"5min":0.33100000000000002,"15min":0.32000000000000001},"max":{"1min":1.21,"5min":1.21,"15min":1.21},"last":0.54100000000000004}]},{"osd":3,"last update":"Fri Sep 13 00:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":0.755,"15min":0.72099999999999997},"min":{"1min":0.63700000000000001,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.0980000000000001,"5min":1.0980000000000001,"15min":1.0980000000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.85299999999999998,"15min":0.79600000000000004},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.8759999999999999,"5min":1.8759999999999999,"15min":1.8759999999999999},"last":0.81499999999999995}]},{"osd":4,"last update":"Fri Sep 13 00:33:20 2024","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.0429999999999999,"15min":0.96699999999999997},"min":{"1min":0.84099999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"max":{"1min":1.2629999999999999,"5min":1.7809999999999999,"15min":1.7809999999999999},"last":1.1040000000000001},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.97799999999999998,"15min":0.88800000000000001},"min":{"1min":0.77900000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.359,"5min":1.633,"15min":1.633},"last":0.78000000000000003}]},{"osd":5,"last update":"Fri Sep 13 00:34:05 2024","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":0.91500000000000004,"15min":0.91000000000000003},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.181,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.94799999999999995,"15min":0.92800000000000005},"min":{"1min":0.621,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.1930000000000001,"5min":1.286,"15min":1.371},"last":0.86399999999999999}]},{"osd":6,"last update":"Fri Sep 13 00:33:58 2024","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":1.0469999999999999,"15min":1.0569999999999999},"min":{"1min":0.69599999999999995,"5min":0.625,"15min":0.625},"max":{"1min":1.921,"5min":1.921,"15min":1.921},"last":1.056},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.01,"15min":1.002},"min":{"1min":0.77300000000000002,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.2210000000000001,"5min":1.823,"15min":1.823},"last":0.98099999999999998}]},{"osd":7,"last update":"Fri Sep 13 00:33:51 2024","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"min":{"1min":0.70999999999999996,"5min":0.65800000000000003,"15min":0.65800000000000003},"max":{"1min":1.2330000000000001,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":1.042},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.111,"15min":1.0920000000000001},"min":{"1min":0.66500000000000004,"5min":0.59399999999999997,"15min":0.59399999999999997},"max":{"1min":2.0070000000000001,"5min":2.0070000000000001,"15min":2.0070000000000001},"last":1.2230000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738513,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298596,"kb_used_data":1236,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446556,"statfs":{"total":95995035648,"available":95689273344,"internally_reserved":0,"allocated":1265664,"data_stored":884992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 00:33:32 2024","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.60999999999999999,"15min":0.54400000000000004},"min":{"1min":0.32600000000000001,"5min":0.317,"15min":0.24399999999999999},"max":{"1min":0.80800000000000005,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68600000000000005,"15min":0.58499999999999996},"min":{"1min":0.38300000000000001,"5min":0.28199999999999997,"15min":0.27400000000000002},"max":{"1min":0.92400000000000004,"5min":1.1799999999999999,"15min":1.2050000000000001},"last":0.83499999999999996}]},{"osd":2,"last update":"Fri Sep 13 00:33:32 2024","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.60599999999999998},"min":{"1min":0.313,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.05,"5min":1.246,"15min":1.246},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.81599999999999995,"5min":0.82499999999999996,"15min":0.73399999999999999},"min":{"1min":0.441,"5min":0.312,"15min":0.312},"max":{"1min":0.98899999999999999,"5min":1.216,"15min":1.216},"last":1.0169999999999999}]},{"osd":3,"last update":"Fri Sep 13 00:33:21 2024","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.76800000000000002,"15min":0.73899999999999999},"min":{"1min":0.36399999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.1259999999999999,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.89600000000000002,"15min":0.81399999999999995},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.2729999999999999,"5min":1.2729999999999999,"15min":1.2729999999999999},"last":0.97299999999999998}]},{"osd":4,"last update":"Fri Sep 13 00:33:17 2024","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.0269999999999999,"15min":0.92500000000000004},"min":{"1min":0.72299999999999998,"5min":0.57799999999999996,"15min":0.57799999999999996},"max":{"1min":1.401,"5min":1.573,"15min":1.573},"last":1.0980000000000001},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.89200000000000002},"min":{"1min":0.58399999999999996,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.1000000000000001,"5min":1.9199999999999999,"15min":1.9199999999999999},"last":0.876}]},{"osd":5,"last update":"Fri Sep 13 00:34:04 2024","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":0.876,"15min":0.86199999999999999},"min":{"1min":0.66200000000000003,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":1.101,"5min":1.272,"15min":1.272},"last":0.752},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.98699999999999999,"15min":0.98199999999999998},"min":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"max":{"1min":1.296,"5min":1.5,"15min":1.5},"last":1.1439999999999999}]},{"osd":6,"last update":"Fri Sep 13 00:33:51 2024","interfaces":[{"interface":"back","average":{"1min":1.032,"5min":1.0209999999999999,"15min":1.0389999999999999},"min":{"1min":0.65600000000000003,"5min":0.57899999999999996,"15min":0.57899999999999996},"max":{"1min":1.3440000000000001,"5min":1.3660000000000001,"15min":1.3660000000000001},"last":0.92900000000000005},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.97599999999999998,"15min":0.94399999999999995},"min":{"1min":0.52900000000000003,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.208,"5min":1.99,"15min":1.99},"last":0.996}]},{"osd":7,"last update":"Fri Sep 13 00:33:42 2024","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.083,"15min":1.119},"min":{"1min":0.64500000000000002,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.278,"5min":1.6319999999999999,"15min":1.6319999999999999},"last":1.0569999999999999},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.0049999999999999,"15min":1.0449999999999999},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.2330000000000001,"5min":1.8280000000000001,"15min":1.8280000000000001},"last":0.95299999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T00:34:23.618 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T00:34:23.618 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T00:34:23.618 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T00:34:23.618 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:34:24.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:23 smithi077 bash[22596]: audit 2024-09-13T00:34:22.836894+0000 mgr.y (mgr.14397) 188 : audit [DBG] from='client.24626 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:34:24.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:23 smithi077 bash[32040]: audit 2024-09-13T00:34:22.836894+0000 mgr.y (mgr.14397) 188 : audit [DBG] from='client.24626 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:34:24.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:23 smithi088 bash[24895]: audit 2024-09-13T00:34:22.836894+0000 mgr.y (mgr.14397) 188 : audit [DBG] from='client.24626 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:34:25.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:24 smithi077 bash[22596]: cluster 2024-09-13T00:34:23.845680+0000 mgr.y (mgr.14397) 189 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:25.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:24 smithi077 bash[32040]: cluster 2024-09-13T00:34:23.845680+0000 mgr.y (mgr.14397) 189 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:25.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:24 smithi088 bash[24895]: cluster 2024-09-13T00:34:23.845680+0000 mgr.y (mgr.14397) 189 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:27.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:26 smithi077 bash[32040]: cluster 2024-09-13T00:34:25.846363+0000 mgr.y (mgr.14397) 190 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:27.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:26 smithi077 bash[22596]: cluster 2024-09-13T00:34:25.846363+0000 mgr.y (mgr.14397) 190 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:27.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:26 smithi088 bash[24895]: cluster 2024-09-13T00:34:25.846363+0000 mgr.y (mgr.14397) 190 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:28.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:27 smithi077 bash[32040]: audit 2024-09-13T00:34:27.834927+0000 mon.c (mon.2) 116 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:28.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:27 smithi077 bash[22596]: audit 2024-09-13T00:34:27.834927+0000 mon.c (mon.2) 116 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:28.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:27 smithi088 bash[24895]: audit 2024-09-13T00:34:27.834927+0000 mon.c (mon.2) 116 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:28.405 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:34:29.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:28 smithi077 bash[22596]: cluster 2024-09-13T00:34:27.847035+0000 mgr.y (mgr.14397) 191 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:29.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:28 smithi077 bash[32040]: cluster 2024-09-13T00:34:27.847035+0000 mgr.y (mgr.14397) 191 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:29.349 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:28 smithi088 bash[24895]: cluster 2024-09-13T00:34:27.847035+0000 mgr.y (mgr.14397) 191 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:31.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:30 smithi088 bash[24895]: cluster 2024-09-13T00:34:29.854435+0000 mgr.y (mgr.14397) 192 : cluster [DBG] pgmap v168: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:31.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:34:30 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:34:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:34:31.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:30 smithi077 bash[22596]: cluster 2024-09-13T00:34:29.854435+0000 mgr.y (mgr.14397) 192 : cluster [DBG] pgmap v168: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:31.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:30 smithi077 bash[32040]: cluster 2024-09-13T00:34:29.854435+0000 mgr.y (mgr.14397) 192 : cluster [DBG] pgmap v168: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:31.774 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:34:31 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:34:31] "GET /metrics HTTP/1.1" 200 240022 "" "Prometheus/2.43.0" 2024-09-13T00:34:32.145 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:34:32.145 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-13T00:34:32.878 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-13T00:34:32.879 INFO:teuthology.run_tasks:Running task print... 2024-09-13T00:34:32.891 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-09-13T00:34:32.891 INFO:teuthology.run_tasks:Running task parallel... 2024-09-13T00:34:32.954 INFO:teuthology.task.parallel:starting parallel... 2024-09-13T00:34:32.955 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T00:34:32.955 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-09-13T00:34:32.960 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-09-13T00:34:32.961 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-13T00:34:32.961 DEBUG:teuthology.orchestra.run.smithi077:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-13T00:34:32.962 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-13T00:34:32.962 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-13T00:34:32.962 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi077.front.sepia.ceph.com 2024-09-13T00:34:32.962 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-09-13T00:34:32.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T00:34:32.970 INFO:teuthology.orchestra.run.smithi077.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-13T00:34:32.971 DEBUG:teuthology.orchestra.run.smithi077:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T00:34:33.020 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-13T00:34:33.020 DEBUG:teuthology.orchestra.run.smithi077:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-13T00:34:33.072 INFO:tasks.workunit:timeout=3h 2024-09-13T00:34:33.072 INFO:tasks.workunit:cleanup=True 2024-09-13T00:34:33.072 DEBUG:teuthology.orchestra.run.smithi077:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-13T00:34:33.122 INFO:tasks.workunit.client.0.smithi077.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-13T00:34:33.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:32 smithi077 bash[22596]: cluster 2024-09-13T00:34:31.854890+0000 mgr.y (mgr.14397) 193 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:33.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:32 smithi077 bash[22596]: audit 2024-09-13T00:34:32.151076+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.77:0/2502161598' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:34:33.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:32 smithi077 bash[32040]: cluster 2024-09-13T00:34:31.854890+0000 mgr.y (mgr.14397) 193 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:33.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:32 smithi077 bash[32040]: audit 2024-09-13T00:34:32.151076+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.77:0/2502161598' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:34:33.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:32 smithi088 bash[24895]: cluster 2024-09-13T00:34:31.854890+0000 mgr.y (mgr.14397) 193 : cluster [DBG] pgmap v169: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:33.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:32 smithi088 bash[24895]: audit 2024-09-13T00:34:32.151076+0000 mon.c (mon.2) 117 : audit [DBG] from='client.? 172.21.15.77:0/2502161598' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:34:35.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:34 smithi077 bash[22596]: cluster 2024-09-13T00:34:33.855850+0000 mgr.y (mgr.14397) 194 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:35.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:34 smithi077 bash[32040]: cluster 2024-09-13T00:34:33.855850+0000 mgr.y (mgr.14397) 194 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:35.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:34 smithi088 bash[24895]: cluster 2024-09-13T00:34:33.855850+0000 mgr.y (mgr.14397) 194 : cluster [DBG] pgmap v170: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:36.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:35 smithi077 bash[22596]: audit 2024-09-13T00:34:35.752366+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:34:36.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:35 smithi077 bash[32040]: audit 2024-09-13T00:34:35.752366+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:34:36.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:35 smithi088 bash[24895]: audit 2024-09-13T00:34:35.752366+0000 mon.c (mon.2) 118 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:34:37.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:36 smithi077 bash[22596]: cluster 2024-09-13T00:34:35.856428+0000 mgr.y (mgr.14397) 195 : cluster [DBG] pgmap v171: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:37.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:36 smithi077 bash[22596]: audit 2024-09-13T00:34:36.213073+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:34:37.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:36 smithi077 bash[22596]: audit 2024-09-13T00:34:36.213840+0000 mon.c (mon.2) 120 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:34:37.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:36 smithi077 bash[22596]: audit 2024-09-13T00:34:36.220376+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:37.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:36 smithi077 bash[32040]: cluster 2024-09-13T00:34:35.856428+0000 mgr.y (mgr.14397) 195 : cluster [DBG] pgmap v171: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:37.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:36 smithi077 bash[32040]: audit 2024-09-13T00:34:36.213073+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:34:37.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:36 smithi077 bash[32040]: audit 2024-09-13T00:34:36.213840+0000 mon.c (mon.2) 120 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:34:37.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:36 smithi077 bash[32040]: audit 2024-09-13T00:34:36.220376+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:37.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:36 smithi088 bash[24895]: cluster 2024-09-13T00:34:35.856428+0000 mgr.y (mgr.14397) 195 : cluster [DBG] pgmap v171: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:37.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:36 smithi088 bash[24895]: audit 2024-09-13T00:34:36.213073+0000 mon.c (mon.2) 119 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:34:37.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:36 smithi088 bash[24895]: audit 2024-09-13T00:34:36.213840+0000 mon.c (mon.2) 120 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:34:37.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:36 smithi088 bash[24895]: audit 2024-09-13T00:34:36.220376+0000 mon.a (mon.0) 818 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:37.741 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:34:39.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:38 smithi077 bash[22596]: cluster 2024-09-13T00:34:37.857003+0000 mgr.y (mgr.14397) 196 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:39.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:38 smithi077 bash[32040]: cluster 2024-09-13T00:34:37.857003+0000 mgr.y (mgr.14397) 196 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:39.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:38 smithi088 bash[24895]: cluster 2024-09-13T00:34:37.857003+0000 mgr.y (mgr.14397) 196 : cluster [DBG] pgmap v172: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:39.710 DEBUG:teuthology.orchestra.run.smithi077:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-13T00:34:39.756 INFO:tasks.workunit.client.0.smithi077.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-13T00:34:39.760 INFO:tasks.workunit.client.0.smithi077.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T00:34:39.760 INFO:tasks.workunit.client.0.smithi077.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-13T00:34:41.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:40 smithi088 bash[24895]: cluster 2024-09-13T00:34:39.858067+0000 mgr.y (mgr.14397) 197 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:41.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:34:40 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:34:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:34:41.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:41 smithi077 bash[22596]: cluster 2024-09-13T00:34:39.858067+0000 mgr.y (mgr.14397) 197 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:41.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:41 smithi077 bash[32040]: cluster 2024-09-13T00:34:39.858067+0000 mgr.y (mgr.14397) 197 : cluster [DBG] pgmap v173: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:41.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:34:41 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:34:41] "GET /metrics HTTP/1.1" 200 240022 "" "Prometheus/2.43.0" 2024-09-13T00:34:43.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:43 smithi077 bash[22596]: cluster 2024-09-13T00:34:41.858633+0000 mgr.y (mgr.14397) 198 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:43.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:43 smithi077 bash[22596]: audit 2024-09-13T00:34:42.835405+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:43.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:43 smithi077 bash[32040]: cluster 2024-09-13T00:34:41.858633+0000 mgr.y (mgr.14397) 198 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:43.275 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:43 smithi077 bash[32040]: audit 2024-09-13T00:34:42.835405+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:43.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:43 smithi088 bash[24895]: cluster 2024-09-13T00:34:41.858633+0000 mgr.y (mgr.14397) 198 : cluster [DBG] pgmap v174: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:43.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:43 smithi088 bash[24895]: audit 2024-09-13T00:34:42.835405+0000 mon.c (mon.2) 121 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:45.192 INFO:tasks.workunit.client.0.smithi077.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-13T00:34:45.274 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:45 smithi077 bash[32040]: cluster 2024-09-13T00:34:43.859601+0000 mgr.y (mgr.14397) 199 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:45.275 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:45 smithi077 bash[22596]: cluster 2024-09-13T00:34:43.859601+0000 mgr.y (mgr.14397) 199 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:45.280 INFO:tasks.workunit.client.0.smithi077.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-13T00:34:45.329 INFO:tasks.workunit.client.0.smithi077.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-13T00:34:45.331 INFO:tasks.workunit.client.0.smithi077.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T00:34:45.331 INFO:tasks.workunit.client.0.smithi077.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-13T00:34:45.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:45 smithi088 bash[24895]: cluster 2024-09-13T00:34:43.859601+0000 mgr.y (mgr.14397) 199 : cluster [DBG] pgmap v175: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:45.369 INFO:tasks.workunit.client.0.smithi077.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-13T00:34:45.372 DEBUG:teuthology.orchestra.run.smithi077:> set -ex 2024-09-13T00:34:45.372 DEBUG:teuthology.orchestra.run.smithi077:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-13T00:34:45.423 INFO:tasks.workunit:Running workunits matching rbd/test_librbd.sh on client.0... 2024-09-13T00:34:45.425 INFO:tasks.workunit:Running workunit rbd/test_librbd.sh... 2024-09-13T00:34:45.425 DEBUG:teuthology.orchestra.run.smithi077:workunit test rbd/test_librbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd.sh 2024-09-13T00:34:45.545 INFO:tasks.workunit.client.0.smithi077.stderr:warning: failed to enable lockdep 2024-09-13T00:34:45.546 INFO:tasks.workunit.client.0.smithi077.stdout:seed 68798 2024-09-13T00:34:45.546 INFO:tasks.workunit.client.0.smithi077.stdout:[==========] Running 406 tests from 15 test suites. 2024-09-13T00:34:45.546 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] Global test environment set-up. 2024-09-13T00:34:45.546 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 16 tests from TestJournalReplay 2024-09-13T00:34:45.574 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:34:45 smithi077 bash[22845]: [13/Sep/2024:00:34:45] ENGINE Bus STOPPING 2024-09-13T00:34:45.575 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:34:45 smithi077 bash[22845]: [13/Sep/2024:00:34:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:34:45.575 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:34:45 smithi077 bash[22845]: [13/Sep/2024:00:34:45] ENGINE Bus STOPPED 2024-09-13T00:34:45.575 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:34:45 smithi077 bash[22845]: [13/Sep/2024:00:34:45] ENGINE Bus STARTING 2024-09-13T00:34:46.025 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:34:45 smithi077 bash[22845]: [13/Sep/2024:00:34:45] ENGINE Serving on http://:::9283 2024-09-13T00:34:46.025 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:34:45 smithi077 bash[22845]: [13/Sep/2024:00:34:45] ENGINE Bus STARTED 2024-09-13T00:34:46.180 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-13T00:34:46.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:46 smithi077 bash[22596]: audit 2024-09-13T00:34:45.280669+0000 mon.a (mon.0) 819 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:34:46.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:46 smithi077 bash[22596]: audit 2024-09-13T00:34:45.291788+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:34:46.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:46 smithi077 bash[32040]: audit 2024-09-13T00:34:45.280669+0000 mon.a (mon.0) 819 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:34:46.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:46 smithi077 bash[32040]: audit 2024-09-13T00:34:45.291788+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:34:46.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:46 smithi088 bash[24895]: audit 2024-09-13T00:34:45.280669+0000 mon.a (mon.0) 819 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:34:46.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:46 smithi088 bash[24895]: audit 2024-09-13T00:34:45.291788+0000 mon.c (mon.2) 122 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:34:47.323 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalReplay.AioDiscardEvent 2024-09-13T00:34:47.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:47 smithi088 bash[24895]: cluster 2024-09-13T00:34:45.860091+0000 mgr.y (mgr.14397) 200 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:47.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:47 smithi088 bash[24895]: cluster 2024-09-13T00:34:46.303886+0000 mon.a (mon.0) 820 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-09-13T00:34:47.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:47 smithi088 bash[24895]: audit 2024-09-13T00:34:46.310761+0000 mon.c (mon.2) 123 : audit [INF] from='client.? 172.21.15.77:0/90184183' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:47.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:47 smithi088 bash[24895]: audit 2024-09-13T00:34:46.311700+0000 mon.a (mon.0) 821 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:47.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:47 smithi077 bash[22596]: cluster 2024-09-13T00:34:45.860091+0000 mgr.y (mgr.14397) 200 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:47.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:47 smithi077 bash[22596]: cluster 2024-09-13T00:34:46.303886+0000 mon.a (mon.0) 820 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-09-13T00:34:47.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:47 smithi077 bash[22596]: audit 2024-09-13T00:34:46.310761+0000 mon.c (mon.2) 123 : audit [INF] from='client.? 172.21.15.77:0/90184183' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:47.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:47 smithi077 bash[22596]: audit 2024-09-13T00:34:46.311700+0000 mon.a (mon.0) 821 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:47.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:47 smithi077 bash[32040]: cluster 2024-09-13T00:34:45.860091+0000 mgr.y (mgr.14397) 200 : cluster [DBG] pgmap v176: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:47.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:47 smithi077 bash[32040]: cluster 2024-09-13T00:34:46.303886+0000 mon.a (mon.0) 820 : cluster [DBG] osdmap e66: 8 total, 8 up, 8 in 2024-09-13T00:34:47.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:47 smithi077 bash[32040]: audit 2024-09-13T00:34:46.310761+0000 mon.c (mon.2) 123 : audit [INF] from='client.? 172.21.15.77:0/90184183' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:47.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:47 smithi077 bash[32040]: audit 2024-09-13T00:34:46.311700+0000 mon.a (mon.0) 821 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:48.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:48 smithi088 bash[24895]: audit 2024-09-13T00:34:47.300716+0000 mon.a (mon.0) 822 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:34:48.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:48 smithi088 bash[24895]: cluster 2024-09-13T00:34:47.301051+0000 mon.a (mon.0) 823 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-09-13T00:34:48.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:48 smithi077 bash[22596]: audit 2024-09-13T00:34:47.300716+0000 mon.a (mon.0) 822 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:34:48.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:48 smithi077 bash[22596]: cluster 2024-09-13T00:34:47.301051+0000 mon.a (mon.0) 823 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-09-13T00:34:48.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:48 smithi077 bash[32040]: audit 2024-09-13T00:34:47.300716+0000 mon.a (mon.0) 822 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:34:48.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:48 smithi077 bash[32040]: cluster 2024-09-13T00:34:47.301051+0000 mon.a (mon.0) 823 : cluster [DBG] osdmap e67: 8 total, 8 up, 8 in 2024-09-13T00:34:49.358 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:49.359 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalReplay.AioDiscardEvent (2035 ms) 2024-09-13T00:34:49.359 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalReplay.AioWriteEvent 2024-09-13T00:34:49.381 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:49.381 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalReplay.AioWriteEvent (23 ms) 2024-09-13T00:34:49.381 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalReplay.AioFlushEvent 2024-09-13T00:34:49.402 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:49.402 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalReplay.AioFlushEvent (20 ms) 2024-09-13T00:34:49.402 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalReplay.SnapCreate 2024-09-13T00:34:49.422 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:49.422 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalReplay.SnapCreate (21 ms) 2024-09-13T00:34:49.423 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalReplay.SnapProtect 2024-09-13T00:34:49.444 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:49.445 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalReplay.SnapProtect (22 ms) 2024-09-13T00:34:49.445 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalReplay.SnapUnprotect 2024-09-13T00:34:49.465 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:49.465 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalReplay.SnapUnprotect (21 ms) 2024-09-13T00:34:49.466 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalReplay.SnapRename 2024-09-13T00:34:49.485 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:49.485 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalReplay.SnapRename (19 ms) 2024-09-13T00:34:49.485 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalReplay.SnapRollback 2024-09-13T00:34:49.506 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:49.506 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalReplay.SnapRollback (22 ms) 2024-09-13T00:34:49.506 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalReplay.SnapRemove 2024-09-13T00:34:49.526 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:49.526 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalReplay.SnapRemove (20 ms) 2024-09-13T00:34:49.526 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalReplay.Rename 2024-09-13T00:34:49.546 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:49.546 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalReplay.Rename (20 ms) 2024-09-13T00:34:49.546 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalReplay.Resize 2024-09-13T00:34:49.566 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:49.566 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalReplay.Resize (20 ms) 2024-09-13T00:34:49.566 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalReplay.Flatten 2024-09-13T00:34:49.581 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:49.581 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalReplay.Flatten (15 ms) 2024-09-13T00:34:49.581 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalReplay.UpdateFeatures 2024-09-13T00:34:49.596 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:49.596 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalReplay.UpdateFeatures (14 ms) 2024-09-13T00:34:49.596 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalReplay.MetadataSet 2024-09-13T00:34:49.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:49 smithi088 bash[24895]: cluster 2024-09-13T00:34:47.860720+0000 mgr.y (mgr.14397) 201 : cluster [DBG] pgmap v179: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:49.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:49 smithi088 bash[24895]: cluster 2024-09-13T00:34:48.330440+0000 mon.a (mon.0) 824 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-09-13T00:34:49.611 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:49.611 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalReplay.MetadataSet (15 ms) 2024-09-13T00:34:49.611 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalReplay.MetadataRemove 2024-09-13T00:34:49.626 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:49.626 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalReplay.MetadataRemove (15 ms) 2024-09-13T00:34:49.626 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalReplay.ObjectPosition 2024-09-13T00:34:49.641 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:49.641 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalReplay.ObjectPosition (15 ms) 2024-09-13T00:34:49.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:49 smithi077 bash[22596]: cluster 2024-09-13T00:34:47.860720+0000 mgr.y (mgr.14397) 201 : cluster [DBG] pgmap v179: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:49.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:49 smithi077 bash[22596]: cluster 2024-09-13T00:34:48.330440+0000 mon.a (mon.0) 824 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-09-13T00:34:49.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:49 smithi077 bash[32040]: cluster 2024-09-13T00:34:47.860720+0000 mgr.y (mgr.14397) 201 : cluster [DBG] pgmap v179: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:49.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:49 smithi077 bash[32040]: cluster 2024-09-13T00:34:48.330440+0000 mon.a (mon.0) 824 : cluster [DBG] osdmap e68: 8 total, 8 up, 8 in 2024-09-13T00:34:50.073 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:34:50.349 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 16 tests from TestJournalReplay (2317 ms total) 2024-09-13T00:34:50.349 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:34:50.350 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 3 tests from TestJournalEntries 2024-09-13T00:34:50.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:50 smithi077 bash[32040]: cluster 2024-09-13T00:34:49.333319+0000 mon.a (mon.0) 825 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-09-13T00:34:50.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:50 smithi077 bash[22596]: cluster 2024-09-13T00:34:49.333319+0000 mon.a (mon.0) 825 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-09-13T00:34:50.599 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:50 smithi088 bash[24895]: cluster 2024-09-13T00:34:49.333319+0000 mon.a (mon.0) 825 : cluster [DBG] osdmap e69: 8 total, 8 up, 8 in 2024-09-13T00:34:51.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:34:50 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:34:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:34:51.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:51 smithi088 bash[24895]: cluster 2024-09-13T00:34:49.861374+0000 mgr.y (mgr.14397) 202 : cluster [DBG] pgmap v182: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T00:34:51.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:51 smithi088 bash[24895]: cluster 2024-09-13T00:34:50.347957+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-09-13T00:34:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:51 smithi088 bash[24895]: audit 2024-09-13T00:34:50.813317+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:51 smithi088 bash[24895]: audit 2024-09-13T00:34:50.822811+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:51 smithi088 bash[24895]: audit 2024-09-13T00:34:50.918428+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:51 smithi088 bash[24895]: audit 2024-09-13T00:34:50.930804+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:51 smithi088 bash[24895]: audit 2024-09-13T00:34:50.935326+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:34:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:51 smithi088 bash[24895]: audit 2024-09-13T00:34:50.937273+0000 mon.c (mon.2) 125 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:34:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:51 smithi088 bash[24895]: audit 2024-09-13T00:34:50.947691+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:51.606 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:34:51] "GET /metrics HTTP/1.1" 200 239992 "" "Prometheus/2.43.0" 2024-09-13T00:34:51.606 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[32040]: cluster 2024-09-13T00:34:49.861374+0000 mgr.y (mgr.14397) 202 : cluster [DBG] pgmap v182: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T00:34:51.606 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[32040]: cluster 2024-09-13T00:34:50.347957+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-09-13T00:34:51.607 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[32040]: audit 2024-09-13T00:34:50.813317+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:51.607 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[32040]: audit 2024-09-13T00:34:50.822811+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:51.607 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[32040]: audit 2024-09-13T00:34:50.918428+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:51.607 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[32040]: audit 2024-09-13T00:34:50.930804+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:51.607 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[32040]: audit 2024-09-13T00:34:50.935326+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:34:51.607 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[32040]: audit 2024-09-13T00:34:50.937273+0000 mon.c (mon.2) 125 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:34:51.607 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[32040]: audit 2024-09-13T00:34:50.947691+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:51.607 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[22596]: cluster 2024-09-13T00:34:49.861374+0000 mgr.y (mgr.14397) 202 : cluster [DBG] pgmap v182: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T00:34:51.607 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[22596]: cluster 2024-09-13T00:34:50.347957+0000 mon.a (mon.0) 826 : cluster [DBG] osdmap e70: 8 total, 8 up, 8 in 2024-09-13T00:34:51.608 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[22596]: audit 2024-09-13T00:34:50.813317+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:51.608 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[22596]: audit 2024-09-13T00:34:50.822811+0000 mon.a (mon.0) 828 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:51.608 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[22596]: audit 2024-09-13T00:34:50.918428+0000 mon.a (mon.0) 829 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:51.608 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[22596]: audit 2024-09-13T00:34:50.930804+0000 mon.a (mon.0) 830 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:51.608 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[22596]: audit 2024-09-13T00:34:50.935326+0000 mon.c (mon.2) 124 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:34:51.608 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[22596]: audit 2024-09-13T00:34:50.937273+0000 mon.c (mon.2) 125 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:34:51.608 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:51 smithi077 bash[22596]: audit 2024-09-13T00:34:50.947691+0000 mon.a (mon.0) 831 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:34:52.374 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalEntries.AioWrite 2024-09-13T00:34:52.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:52 smithi077 bash[22596]: cluster 2024-09-13T00:34:51.363212+0000 mon.a (mon.0) 832 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-09-13T00:34:52.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:52 smithi077 bash[22596]: audit 2024-09-13T00:34:51.365418+0000 mon.c (mon.2) 126 : audit [INF] from='client.? 172.21.15.77:0/1090777167' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:52.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:52 smithi077 bash[22596]: audit 2024-09-13T00:34:51.367401+0000 mon.a (mon.0) 833 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:52.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:52 smithi077 bash[32040]: cluster 2024-09-13T00:34:51.363212+0000 mon.a (mon.0) 832 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-09-13T00:34:52.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:52 smithi077 bash[32040]: audit 2024-09-13T00:34:51.365418+0000 mon.c (mon.2) 126 : audit [INF] from='client.? 172.21.15.77:0/1090777167' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:52.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:52 smithi077 bash[32040]: audit 2024-09-13T00:34:51.367401+0000 mon.a (mon.0) 833 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:52.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:52 smithi088 bash[24895]: cluster 2024-09-13T00:34:51.363212+0000 mon.a (mon.0) 832 : cluster [DBG] osdmap e71: 8 total, 8 up, 8 in 2024-09-13T00:34:52.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:52 smithi088 bash[24895]: audit 2024-09-13T00:34:51.365418+0000 mon.c (mon.2) 126 : audit [INF] from='client.? 172.21.15.77:0/1090777167' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:52.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:52 smithi088 bash[24895]: audit 2024-09-13T00:34:51.367401+0000 mon.a (mon.0) 833 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:53.725 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:53 smithi077 bash[22596]: cluster 2024-09-13T00:34:51.862055+0000 mgr.y (mgr.14397) 203 : cluster [DBG] pgmap v185: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:53.725 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:53 smithi077 bash[22596]: cluster 2024-09-13T00:34:52.354906+0000 mon.a (mon.0) 834 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:34:53.725 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:53 smithi077 bash[22596]: audit 2024-09-13T00:34:52.363235+0000 mon.a (mon.0) 835 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:34:53.726 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:53 smithi077 bash[22596]: cluster 2024-09-13T00:34:52.363736+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-09-13T00:34:53.726 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:53 smithi077 bash[32040]: cluster 2024-09-13T00:34:51.862055+0000 mgr.y (mgr.14397) 203 : cluster [DBG] pgmap v185: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:53.726 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:53 smithi077 bash[32040]: cluster 2024-09-13T00:34:52.354906+0000 mon.a (mon.0) 834 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:34:53.726 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:53 smithi077 bash[32040]: audit 2024-09-13T00:34:52.363235+0000 mon.a (mon.0) 835 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:34:53.726 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:53 smithi077 bash[32040]: cluster 2024-09-13T00:34:52.363736+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-09-13T00:34:53.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:53 smithi088 bash[24895]: cluster 2024-09-13T00:34:51.862055+0000 mgr.y (mgr.14397) 203 : cluster [DBG] pgmap v185: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:53.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:53 smithi088 bash[24895]: cluster 2024-09-13T00:34:52.354906+0000 mon.a (mon.0) 834 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:34:53.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:53 smithi088 bash[24895]: audit 2024-09-13T00:34:52.363235+0000 mon.a (mon.0) 835 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:34:53.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:53 smithi088 bash[24895]: cluster 2024-09-13T00:34:52.363736+0000 mon.a (mon.0) 836 : cluster [DBG] osdmap e72: 8 total, 8 up, 8 in 2024-09-13T00:34:54.423 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:54.423 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalEntries.AioWrite (2048 ms) 2024-09-13T00:34:54.423 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalEntries.AioDiscard 2024-09-13T00:34:54.442 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:54.442 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalEntries.AioDiscard (20 ms) 2024-09-13T00:34:54.442 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestJournalEntries.AioFlush 2024-09-13T00:34:54.463 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:34:54.463 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestJournalEntries.AioFlush (20 ms) 2024-09-13T00:34:54.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:54 smithi077 bash[22596]: cluster 2024-09-13T00:34:53.376276+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-09-13T00:34:54.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:54 smithi077 bash[32040]: cluster 2024-09-13T00:34:53.376276+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-09-13T00:34:54.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:54 smithi088 bash[24895]: cluster 2024-09-13T00:34:53.376276+0000 mon.a (mon.0) 837 : cluster [DBG] osdmap e73: 8 total, 8 up, 8 in 2024-09-13T00:34:54.874 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-09-13T00:34:55.400 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 3 tests from TestJournalEntries (2088 ms total) 2024-09-13T00:34:55.400 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:34:55.400 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 2 tests from TestTrash 2024-09-13T00:34:55.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:55 smithi077 bash[22596]: cluster 2024-09-13T00:34:53.862756+0000 mgr.y (mgr.14397) 204 : cluster [DBG] pgmap v188: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:55.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:55 smithi077 bash[22596]: cluster 2024-09-13T00:34:54.379795+0000 mon.a (mon.0) 838 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:34:55.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:55 smithi077 bash[22596]: cluster 2024-09-13T00:34:54.379843+0000 mon.a (mon.0) 839 : cluster [INF] Cluster is now healthy 2024-09-13T00:34:55.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:55 smithi077 bash[22596]: cluster 2024-09-13T00:34:54.397123+0000 mon.a (mon.0) 840 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-09-13T00:34:55.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:55 smithi077 bash[32040]: cluster 2024-09-13T00:34:53.862756+0000 mgr.y (mgr.14397) 204 : cluster [DBG] pgmap v188: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:55.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:55 smithi077 bash[32040]: cluster 2024-09-13T00:34:54.379795+0000 mon.a (mon.0) 838 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:34:55.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:55 smithi077 bash[32040]: cluster 2024-09-13T00:34:54.379843+0000 mon.a (mon.0) 839 : cluster [INF] Cluster is now healthy 2024-09-13T00:34:55.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:55 smithi077 bash[32040]: cluster 2024-09-13T00:34:54.397123+0000 mon.a (mon.0) 840 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-09-13T00:34:55.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:55 smithi088 bash[24895]: cluster 2024-09-13T00:34:53.862756+0000 mgr.y (mgr.14397) 204 : cluster [DBG] pgmap v188: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:55.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:55 smithi088 bash[24895]: cluster 2024-09-13T00:34:54.379795+0000 mon.a (mon.0) 838 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:34:55.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:55 smithi088 bash[24895]: cluster 2024-09-13T00:34:54.379843+0000 mon.a (mon.0) 839 : cluster [INF] Cluster is now healthy 2024-09-13T00:34:55.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:55 smithi088 bash[24895]: cluster 2024-09-13T00:34:54.397123+0000 mon.a (mon.0) 840 : cluster [DBG] osdmap e74: 8 total, 8 up, 8 in 2024-09-13T00:34:56.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:56 smithi077 bash[22596]: cluster 2024-09-13T00:34:55.399657+0000 mon.a (mon.0) 841 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-09-13T00:34:56.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:56 smithi077 bash[32040]: cluster 2024-09-13T00:34:55.399657+0000 mon.a (mon.0) 841 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-09-13T00:34:56.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:56 smithi088 bash[24895]: cluster 2024-09-13T00:34:55.399657+0000 mon.a (mon.0) 841 : cluster [DBG] osdmap e75: 8 total, 8 up, 8 in 2024-09-13T00:34:57.426 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestTrash.UserRemovingSource 2024-09-13T00:34:57.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:57 smithi077 bash[32040]: cluster 2024-09-13T00:34:55.863753+0000 mgr.y (mgr.14397) 205 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:57.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:57 smithi077 bash[32040]: cluster 2024-09-13T00:34:56.428561+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-09-13T00:34:57.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:57 smithi077 bash[32040]: audit 2024-09-13T00:34:56.434708+0000 mon.a (mon.0) 843 : audit [INF] from='client.? 172.21.15.77:0/1523019149' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:57.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:57 smithi077 bash[22596]: cluster 2024-09-13T00:34:55.863753+0000 mgr.y (mgr.14397) 205 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:57.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:57 smithi077 bash[22596]: cluster 2024-09-13T00:34:56.428561+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-09-13T00:34:57.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:57 smithi077 bash[22596]: audit 2024-09-13T00:34:56.434708+0000 mon.a (mon.0) 843 : audit [INF] from='client.? 172.21.15.77:0/1523019149' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:57.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:57 smithi088 bash[24895]: cluster 2024-09-13T00:34:55.863753+0000 mgr.y (mgr.14397) 205 : cluster [DBG] pgmap v191: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:57.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:57 smithi088 bash[24895]: cluster 2024-09-13T00:34:56.428561+0000 mon.a (mon.0) 842 : cluster [DBG] osdmap e76: 8 total, 8 up, 8 in 2024-09-13T00:34:57.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:57 smithi088 bash[24895]: audit 2024-09-13T00:34:56.434708+0000 mon.a (mon.0) 843 : audit [INF] from='client.? 172.21.15.77:0/1523019149' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:34:58.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:58 smithi077 bash[32040]: audit 2024-09-13T00:34:57.429284+0000 mon.a (mon.0) 844 : audit [INF] from='client.? 172.21.15.77:0/1523019149' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:34:58.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:58 smithi077 bash[32040]: cluster 2024-09-13T00:34:57.430383+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2024-09-13T00:34:58.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:58 smithi077 bash[32040]: audit 2024-09-13T00:34:57.835726+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:58.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:58 smithi077 bash[22596]: audit 2024-09-13T00:34:57.429284+0000 mon.a (mon.0) 844 : audit [INF] from='client.? 172.21.15.77:0/1523019149' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:34:58.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:58 smithi077 bash[22596]: cluster 2024-09-13T00:34:57.430383+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2024-09-13T00:34:58.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:58 smithi077 bash[22596]: audit 2024-09-13T00:34:57.835726+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:58.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:58 smithi088 bash[24895]: audit 2024-09-13T00:34:57.429284+0000 mon.a (mon.0) 844 : audit [INF] from='client.? 172.21.15.77:0/1523019149' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:34:58.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:58 smithi088 bash[24895]: cluster 2024-09-13T00:34:57.430383+0000 mon.a (mon.0) 845 : cluster [DBG] osdmap e77: 8 total, 8 up, 8 in 2024-09-13T00:34:58.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:58 smithi088 bash[24895]: audit 2024-09-13T00:34:57.835726+0000 mon.c (mon.2) 127 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:34:59.651 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestTrash.UserRemovingSource (2225 ms) 2024-09-13T00:34:59.651 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestTrash.RestoreMirroringSource 2024-09-13T00:34:59.659 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:34:59.718 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestTrash.RestoreMirroringSource (66 ms) 2024-09-13T00:34:59.742 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:59 smithi077 bash[22596]: cluster 2024-09-13T00:34:57.864944+0000 mgr.y (mgr.14397) 206 : cluster [DBG] pgmap v194: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:59.742 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:34:59 smithi077 bash[22596]: cluster 2024-09-13T00:34:58.445741+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-09-13T00:34:59.742 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:59 smithi077 bash[32040]: cluster 2024-09-13T00:34:57.864944+0000 mgr.y (mgr.14397) 206 : cluster [DBG] pgmap v194: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:59.742 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:34:59 smithi077 bash[32040]: cluster 2024-09-13T00:34:58.445741+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-09-13T00:34:59.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:59 smithi088 bash[24895]: cluster 2024-09-13T00:34:57.864944+0000 mgr.y (mgr.14397) 206 : cluster [DBG] pgmap v194: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:34:59.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:34:59 smithi088 bash[24895]: cluster 2024-09-13T00:34:58.445741+0000 mon.a (mon.0) 846 : cluster [DBG] osdmap e78: 8 total, 8 up, 8 in 2024-09-13T00:35:00.480 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 2 tests from TestTrash (2291 ms total) 2024-09-13T00:35:00.480 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:35:00.480 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 1 test from TestOperations 2024-09-13T00:35:00.760 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:00 smithi088 bash[24895]: cluster 2024-09-13T00:34:59.467091+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-09-13T00:35:00.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:00 smithi077 bash[22596]: cluster 2024-09-13T00:34:59.467091+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-09-13T00:35:00.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:00 smithi077 bash[32040]: cluster 2024-09-13T00:34:59.467091+0000 mon.a (mon.0) 847 : cluster [DBG] osdmap e79: 8 total, 8 up, 8 in 2024-09-13T00:35:01.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:35:00 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:35:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:35:01.634 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:01 smithi077 bash[22596]: cluster 2024-09-13T00:34:59.865638+0000 mgr.y (mgr.14397) 207 : cluster [DBG] pgmap v197: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T00:35:01.635 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:01 smithi077 bash[22596]: cluster 2024-09-13T00:35:00.478130+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2024-09-13T00:35:01.635 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:35:01 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:35:01] "GET /metrics HTTP/1.1" 200 242692 "" "Prometheus/2.43.0" 2024-09-13T00:35:01.635 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:01 smithi077 bash[32040]: cluster 2024-09-13T00:34:59.865638+0000 mgr.y (mgr.14397) 207 : cluster [DBG] pgmap v197: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T00:35:01.635 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:01 smithi077 bash[32040]: cluster 2024-09-13T00:35:00.478130+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2024-09-13T00:35:01.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:01 smithi088 bash[24895]: cluster 2024-09-13T00:34:59.865638+0000 mgr.y (mgr.14397) 207 : cluster [DBG] pgmap v197: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T00:35:01.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:01 smithi088 bash[24895]: cluster 2024-09-13T00:35:00.478130+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e80: 8 total, 8 up, 8 in 2024-09-13T00:35:02.500 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestOperations.DisableJournalingCorrupt 2024-09-13T00:35:02.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:02 smithi077 bash[22596]: cluster 2024-09-13T00:35:01.490907+0000 mon.a (mon.0) 849 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2024-09-13T00:35:02.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:02 smithi077 bash[22596]: audit 2024-09-13T00:35:01.494812+0000 mon.c (mon.2) 128 : audit [INF] from='client.? 172.21.15.77:0/3942965995' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:02.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:02 smithi077 bash[22596]: audit 2024-09-13T00:35:01.495893+0000 mon.a (mon.0) 850 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:02.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:02 smithi077 bash[32040]: cluster 2024-09-13T00:35:01.490907+0000 mon.a (mon.0) 849 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2024-09-13T00:35:02.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:02 smithi077 bash[32040]: audit 2024-09-13T00:35:01.494812+0000 mon.c (mon.2) 128 : audit [INF] from='client.? 172.21.15.77:0/3942965995' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:02.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:02 smithi077 bash[32040]: audit 2024-09-13T00:35:01.495893+0000 mon.a (mon.0) 850 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:02.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:02 smithi088 bash[24895]: cluster 2024-09-13T00:35:01.490907+0000 mon.a (mon.0) 849 : cluster [DBG] osdmap e81: 8 total, 8 up, 8 in 2024-09-13T00:35:02.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:02 smithi088 bash[24895]: audit 2024-09-13T00:35:01.494812+0000 mon.c (mon.2) 128 : audit [INF] from='client.? 172.21.15.77:0/3942965995' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:02.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:02 smithi088 bash[24895]: audit 2024-09-13T00:35:01.495893+0000 mon.a (mon.0) 850 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:03.757 INFO:teuthology.orchestra.run.smithi077.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-13T00:35:03.770 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:03 smithi077 bash[32040]: cluster 2024-09-13T00:35:01.866323+0000 mgr.y (mgr.14397) 208 : cluster [DBG] pgmap v200: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:03.770 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:03 smithi077 bash[32040]: cluster 2024-09-13T00:35:02.483756+0000 mon.a (mon.0) 851 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:35:03.771 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:03 smithi077 bash[32040]: audit 2024-09-13T00:35:02.502034+0000 mon.a (mon.0) 852 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:03.771 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:03 smithi077 bash[32040]: cluster 2024-09-13T00:35:02.502817+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2024-09-13T00:35:03.771 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:03 smithi077 bash[22596]: cluster 2024-09-13T00:35:01.866323+0000 mgr.y (mgr.14397) 208 : cluster [DBG] pgmap v200: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:03.771 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:03 smithi077 bash[22596]: cluster 2024-09-13T00:35:02.483756+0000 mon.a (mon.0) 851 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:35:03.771 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:03 smithi077 bash[22596]: audit 2024-09-13T00:35:02.502034+0000 mon.a (mon.0) 852 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:03.771 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:03 smithi077 bash[22596]: cluster 2024-09-13T00:35:02.502817+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2024-09-13T00:35:03.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:03 smithi088 bash[24895]: cluster 2024-09-13T00:35:01.866323+0000 mgr.y (mgr.14397) 208 : cluster [DBG] pgmap v200: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:03.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:03 smithi088 bash[24895]: cluster 2024-09-13T00:35:02.483756+0000 mon.a (mon.0) 851 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:35:03.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:03 smithi088 bash[24895]: audit 2024-09-13T00:35:02.502034+0000 mon.a (mon.0) 852 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:03.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:03 smithi088 bash[24895]: cluster 2024-09-13T00:35:02.502817+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e82: 8 total, 8 up, 8 in 2024-09-13T00:35:04.556 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:35:04.556 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestOperations.DisableJournalingCorrupt (2056 ms) 2024-09-13T00:35:04.628 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph orch ps' 2024-09-13T00:35:04.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:04 smithi077 bash[22596]: cluster 2024-09-13T00:35:03.504530+0000 mon.a (mon.0) 854 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2024-09-13T00:35:04.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:04 smithi077 bash[22596]: audit 2024-09-13T00:35:03.762093+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:04.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:04 smithi077 bash[22596]: audit 2024-09-13T00:35:03.763584+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:35:04.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:04 smithi077 bash[22596]: audit 2024-09-13T00:35:03.766154+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:35:04.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:04 smithi077 bash[22596]: audit 2024-09-13T00:35:03.767170+0000 mon.c (mon.2) 131 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:35:04.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:04 smithi077 bash[22596]: audit 2024-09-13T00:35:03.773670+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:04.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:04 smithi077 bash[32040]: cluster 2024-09-13T00:35:03.504530+0000 mon.a (mon.0) 854 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2024-09-13T00:35:04.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:04 smithi077 bash[32040]: audit 2024-09-13T00:35:03.762093+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:04.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:04 smithi077 bash[32040]: audit 2024-09-13T00:35:03.763584+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:35:04.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:04 smithi077 bash[32040]: audit 2024-09-13T00:35:03.766154+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:35:04.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:04 smithi077 bash[32040]: audit 2024-09-13T00:35:03.767170+0000 mon.c (mon.2) 131 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:35:04.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:04 smithi077 bash[32040]: audit 2024-09-13T00:35:03.773670+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:04.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:04 smithi088 bash[24895]: cluster 2024-09-13T00:35:03.504530+0000 mon.a (mon.0) 854 : cluster [DBG] osdmap e83: 8 total, 8 up, 8 in 2024-09-13T00:35:04.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:04 smithi088 bash[24895]: audit 2024-09-13T00:35:03.762093+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:04.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:04 smithi088 bash[24895]: audit 2024-09-13T00:35:03.763584+0000 mon.c (mon.2) 129 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:35:04.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:04 smithi088 bash[24895]: audit 2024-09-13T00:35:03.766154+0000 mon.c (mon.2) 130 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:35:04.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:04 smithi088 bash[24895]: audit 2024-09-13T00:35:03.767170+0000 mon.c (mon.2) 131 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:35:04.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:04 smithi088 bash[24895]: audit 2024-09-13T00:35:03.773670+0000 mon.a (mon.0) 856 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:05.522 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 1 test from TestOperations (2056 ms total) 2024-09-13T00:35:05.522 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:35:05.522 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 4 tests from TestObjectMap 2024-09-13T00:35:05.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:05 smithi088 bash[24895]: audit 2024-09-13T00:35:03.752887+0000 mgr.y (mgr.14397) 209 : audit [DBG] from='client.24704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:35:05.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:05 smithi088 bash[24895]: cephadm 2024-09-13T00:35:03.753981+0000 mgr.y (mgr.14397) 210 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-13T00:35:05.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:05 smithi088 bash[24895]: cephadm 2024-09-13T00:35:03.813911+0000 mgr.y (mgr.14397) 211 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-13T00:35:05.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:05 smithi088 bash[24895]: cluster 2024-09-13T00:35:03.866954+0000 mgr.y (mgr.14397) 212 : cluster [DBG] pgmap v203: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:05.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:05 smithi088 bash[24895]: cluster 2024-09-13T00:35:04.519594+0000 mon.a (mon.0) 857 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-09-13T00:35:05.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:05 smithi088 bash[24895]: cluster 2024-09-13T00:35:04.771193+0000 mon.a (mon.0) 858 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:35:05.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:05 smithi088 bash[24895]: cluster 2024-09-13T00:35:04.771249+0000 mon.a (mon.0) 859 : cluster [INF] Cluster is now healthy 2024-09-13T00:35:06.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:05 smithi077 bash[22596]: audit 2024-09-13T00:35:03.752887+0000 mgr.y (mgr.14397) 209 : audit [DBG] from='client.24704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:35:06.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:05 smithi077 bash[22596]: cephadm 2024-09-13T00:35:03.753981+0000 mgr.y (mgr.14397) 210 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-13T00:35:06.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:05 smithi077 bash[22596]: cephadm 2024-09-13T00:35:03.813911+0000 mgr.y (mgr.14397) 211 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-13T00:35:06.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:05 smithi077 bash[22596]: cluster 2024-09-13T00:35:03.866954+0000 mgr.y (mgr.14397) 212 : cluster [DBG] pgmap v203: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:06.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:05 smithi077 bash[22596]: cluster 2024-09-13T00:35:04.519594+0000 mon.a (mon.0) 857 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-09-13T00:35:06.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:05 smithi077 bash[22596]: cluster 2024-09-13T00:35:04.771193+0000 mon.a (mon.0) 858 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:35:06.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:05 smithi077 bash[22596]: cluster 2024-09-13T00:35:04.771249+0000 mon.a (mon.0) 859 : cluster [INF] Cluster is now healthy 2024-09-13T00:35:06.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:05 smithi077 bash[32040]: audit 2024-09-13T00:35:03.752887+0000 mgr.y (mgr.14397) 209 : audit [DBG] from='client.24704 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:35:06.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:05 smithi077 bash[32040]: cephadm 2024-09-13T00:35:03.753981+0000 mgr.y (mgr.14397) 210 : cephadm [INF] Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-13T00:35:06.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:05 smithi077 bash[32040]: cephadm 2024-09-13T00:35:03.813911+0000 mgr.y (mgr.14397) 211 : cephadm [INF] Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:7a853b1706d076d58e0da7a5d3b9b9eb36276880 2024-09-13T00:35:06.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:05 smithi077 bash[32040]: cluster 2024-09-13T00:35:03.866954+0000 mgr.y (mgr.14397) 212 : cluster [DBG] pgmap v203: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:06.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:05 smithi077 bash[32040]: cluster 2024-09-13T00:35:04.519594+0000 mon.a (mon.0) 857 : cluster [DBG] osdmap e84: 8 total, 8 up, 8 in 2024-09-13T00:35:06.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:05 smithi077 bash[32040]: cluster 2024-09-13T00:35:04.771193+0000 mon.a (mon.0) 858 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:35:06.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:05 smithi077 bash[32040]: cluster 2024-09-13T00:35:04.771249+0000 mon.a (mon.0) 859 : cluster [INF] Cluster is now healthy 2024-09-13T00:35:06.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:06 smithi088 bash[24895]: cluster 2024-09-13T00:35:05.523423+0000 mon.a (mon.0) 860 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2024-09-13T00:35:07.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:06 smithi077 bash[22596]: cluster 2024-09-13T00:35:05.523423+0000 mon.a (mon.0) 860 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2024-09-13T00:35:07.025 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:06 smithi077 bash[32040]: cluster 2024-09-13T00:35:05.523423+0000 mon.a (mon.0) 860 : cluster [DBG] osdmap e85: 8 total, 8 up, 8 in 2024-09-13T00:35:07.580 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenCorrupt 2024-09-13T00:35:07.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:07 smithi088 bash[24895]: cluster 2024-09-13T00:35:05.867530+0000 mgr.y (mgr.14397) 213 : cluster [DBG] pgmap v206: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:07.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:07 smithi088 bash[24895]: cluster 2024-09-13T00:35:06.550940+0000 mon.a (mon.0) 861 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2024-09-13T00:35:07.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:07 smithi088 bash[24895]: audit 2024-09-13T00:35:06.559563+0000 mon.c (mon.2) 132 : audit [INF] from='client.? 172.21.15.77:0/3261957013' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:07.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:07 smithi088 bash[24895]: audit 2024-09-13T00:35:06.560404+0000 mon.a (mon.0) 862 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:08.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:07 smithi077 bash[22596]: cluster 2024-09-13T00:35:05.867530+0000 mgr.y (mgr.14397) 213 : cluster [DBG] pgmap v206: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:08.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:07 smithi077 bash[22596]: cluster 2024-09-13T00:35:06.550940+0000 mon.a (mon.0) 861 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2024-09-13T00:35:08.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:07 smithi077 bash[22596]: audit 2024-09-13T00:35:06.559563+0000 mon.c (mon.2) 132 : audit [INF] from='client.? 172.21.15.77:0/3261957013' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:08.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:07 smithi077 bash[22596]: audit 2024-09-13T00:35:06.560404+0000 mon.a (mon.0) 862 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:08.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:07 smithi077 bash[32040]: cluster 2024-09-13T00:35:05.867530+0000 mgr.y (mgr.14397) 213 : cluster [DBG] pgmap v206: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:08.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:07 smithi077 bash[32040]: cluster 2024-09-13T00:35:06.550940+0000 mon.a (mon.0) 861 : cluster [DBG] osdmap e86: 8 total, 8 up, 8 in 2024-09-13T00:35:08.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:07 smithi077 bash[32040]: audit 2024-09-13T00:35:06.559563+0000 mon.c (mon.2) 132 : audit [INF] from='client.? 172.21.15.77:0/3261957013' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:08.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:07 smithi077 bash[32040]: audit 2024-09-13T00:35:06.560404+0000 mon.a (mon.0) 862 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:08.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:08 smithi088 bash[24895]: audit 2024-09-13T00:35:07.554821+0000 mon.a (mon.0) 863 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:08.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:08 smithi088 bash[24895]: cluster 2024-09-13T00:35:07.555174+0000 mon.a (mon.0) 864 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2024-09-13T00:35:08.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:08 smithi088 bash[24895]: cluster 2024-09-13T00:35:07.868301+0000 mgr.y (mgr.14397) 214 : cluster [DBG] pgmap v209: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:08.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:08 smithi088 bash[24895]: cluster 2024-09-13T00:35:08.558177+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2024-09-13T00:35:09.025 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:08 smithi077 bash[32040]: audit 2024-09-13T00:35:07.554821+0000 mon.a (mon.0) 863 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:09.025 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:08 smithi077 bash[32040]: cluster 2024-09-13T00:35:07.555174+0000 mon.a (mon.0) 864 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2024-09-13T00:35:09.025 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:08 smithi077 bash[32040]: cluster 2024-09-13T00:35:07.868301+0000 mgr.y (mgr.14397) 214 : cluster [DBG] pgmap v209: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:09.025 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:08 smithi077 bash[32040]: cluster 2024-09-13T00:35:08.558177+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2024-09-13T00:35:09.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:08 smithi077 bash[22596]: audit 2024-09-13T00:35:07.554821+0000 mon.a (mon.0) 863 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:09.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:08 smithi077 bash[22596]: cluster 2024-09-13T00:35:07.555174+0000 mon.a (mon.0) 864 : cluster [DBG] osdmap e87: 8 total, 8 up, 8 in 2024-09-13T00:35:09.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:08 smithi077 bash[22596]: cluster 2024-09-13T00:35:07.868301+0000 mgr.y (mgr.14397) 214 : cluster [DBG] pgmap v209: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:09.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:08 smithi077 bash[22596]: cluster 2024-09-13T00:35:08.558177+0000 mon.a (mon.0) 865 : cluster [DBG] osdmap e88: 8 total, 8 up, 8 in 2024-09-13T00:35:09.411 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:35:09.612 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenCorrupt (2031 ms) 2024-09-13T00:35:09.612 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenTooSmall 2024-09-13T00:35:09.656 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenTooSmall (44 ms) 2024-09-13T00:35:09.656 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestObjectMap.InvalidateFlagOnDisk 2024-09-13T00:35:09.725 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestObjectMap.InvalidateFlagOnDisk (69 ms) 2024-09-13T00:35:09.725 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall 2024-09-13T00:35:09.773 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall (49 ms) 2024-09-13T00:35:10.590 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 4 tests from TestObjectMap (2193 ms total) 2024-09-13T00:35:10.590 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:35:10.590 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 2 tests from TestMirroringWatcher 2024-09-13T00:35:10.850 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:35:10 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:35:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:35:10.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:10 smithi088 bash[24895]: cluster 2024-09-13T00:35:09.560459+0000 mon.a (mon.0) 866 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-09-13T00:35:10.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:10 smithi088 bash[24895]: cluster 2024-09-13T00:35:09.868868+0000 mgr.y (mgr.14397) 215 : cluster [DBG] pgmap v212: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T00:35:11.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:10 smithi077 bash[22596]: cluster 2024-09-13T00:35:09.560459+0000 mon.a (mon.0) 866 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-09-13T00:35:11.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:10 smithi077 bash[22596]: cluster 2024-09-13T00:35:09.868868+0000 mgr.y (mgr.14397) 215 : cluster [DBG] pgmap v212: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T00:35:11.025 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:10 smithi077 bash[32040]: cluster 2024-09-13T00:35:09.560459+0000 mon.a (mon.0) 866 : cluster [DBG] osdmap e89: 8 total, 8 up, 8 in 2024-09-13T00:35:11.025 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:10 smithi077 bash[32040]: cluster 2024-09-13T00:35:09.868868+0000 mgr.y (mgr.14397) 215 : cluster [DBG] pgmap v212: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-13T00:35:11.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:35:11 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:35:11] "GET /metrics HTTP/1.1" 200 242692 "" "Prometheus/2.43.0" 2024-09-13T00:35:11.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:11 smithi077 bash[32040]: cluster 2024-09-13T00:35:10.591699+0000 mon.a (mon.0) 867 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2024-09-13T00:35:11.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:11 smithi077 bash[22596]: cluster 2024-09-13T00:35:10.591699+0000 mon.a (mon.0) 867 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2024-09-13T00:35:11.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:11 smithi088 bash[24895]: cluster 2024-09-13T00:35:10.591699+0000 mon.a (mon.0) 867 : cluster [DBG] osdmap e90: 8 total, 8 up, 8 in 2024-09-13T00:35:12.592 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroringWatcher.ModeUpdated 2024-09-13T00:35:12.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:12 smithi088 bash[24895]: cluster 2024-09-13T00:35:11.586775+0000 mon.a (mon.0) 868 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2024-09-13T00:35:12.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:12 smithi088 bash[24895]: audit 2024-09-13T00:35:11.591864+0000 mon.a (mon.0) 869 : audit [INF] from='client.? 172.21.15.77:0/1079091314' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:12.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:12 smithi088 bash[24895]: cluster 2024-09-13T00:35:11.869730+0000 mgr.y (mgr.14397) 216 : cluster [DBG] pgmap v215: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:13.025 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:12 smithi077 bash[32040]: cluster 2024-09-13T00:35:11.586775+0000 mon.a (mon.0) 868 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2024-09-13T00:35:13.025 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:12 smithi077 bash[32040]: audit 2024-09-13T00:35:11.591864+0000 mon.a (mon.0) 869 : audit [INF] from='client.? 172.21.15.77:0/1079091314' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:13.025 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:12 smithi077 bash[32040]: cluster 2024-09-13T00:35:11.869730+0000 mgr.y (mgr.14397) 216 : cluster [DBG] pgmap v215: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:13.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:12 smithi077 bash[22596]: cluster 2024-09-13T00:35:11.586775+0000 mon.a (mon.0) 868 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2024-09-13T00:35:13.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:12 smithi077 bash[22596]: audit 2024-09-13T00:35:11.591864+0000 mon.a (mon.0) 869 : audit [INF] from='client.? 172.21.15.77:0/1079091314' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:13.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:12 smithi077 bash[22596]: cluster 2024-09-13T00:35:11.869730+0000 mgr.y (mgr.14397) 216 : cluster [DBG] pgmap v215: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:13.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:13 smithi088 bash[24895]: cluster 2024-09-13T00:35:12.574780+0000 mon.a (mon.0) 870 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:35:13.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:13 smithi088 bash[24895]: audit 2024-09-13T00:35:12.592829+0000 mon.a (mon.0) 871 : audit [INF] from='client.? 172.21.15.77:0/1079091314' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:13.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:13 smithi088 bash[24895]: cluster 2024-09-13T00:35:12.593275+0000 mon.a (mon.0) 872 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-09-13T00:35:13.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:13 smithi088 bash[24895]: audit 2024-09-13T00:35:12.844388+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:13.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:13 smithi088 bash[24895]: audit 2024-09-13T00:35:12.846189+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:14.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:13 smithi077 bash[22596]: cluster 2024-09-13T00:35:12.574780+0000 mon.a (mon.0) 870 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:35:14.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:13 smithi077 bash[22596]: audit 2024-09-13T00:35:12.592829+0000 mon.a (mon.0) 871 : audit [INF] from='client.? 172.21.15.77:0/1079091314' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:14.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:13 smithi077 bash[22596]: cluster 2024-09-13T00:35:12.593275+0000 mon.a (mon.0) 872 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-09-13T00:35:14.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:13 smithi077 bash[22596]: audit 2024-09-13T00:35:12.844388+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:14.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:13 smithi077 bash[22596]: audit 2024-09-13T00:35:12.846189+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:14.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:13 smithi077 bash[32040]: cluster 2024-09-13T00:35:12.574780+0000 mon.a (mon.0) 870 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:35:14.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:13 smithi077 bash[32040]: audit 2024-09-13T00:35:12.592829+0000 mon.a (mon.0) 871 : audit [INF] from='client.? 172.21.15.77:0/1079091314' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:14.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:13 smithi077 bash[32040]: cluster 2024-09-13T00:35:12.593275+0000 mon.a (mon.0) 872 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-09-13T00:35:14.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:13 smithi077 bash[32040]: audit 2024-09-13T00:35:12.844388+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:14.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:13 smithi077 bash[32040]: audit 2024-09-13T00:35:12.846189+0000 mon.c (mon.2) 133 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:14.668 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroringWatcher.ModeUpdated (2076 ms) 2024-09-13T00:35:14.668 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroringWatcher.ImageStatusUpdated 2024-09-13T00:35:14.719 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroringWatcher.ImageStatusUpdated (50 ms) 2024-09-13T00:35:14.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:14 smithi088 bash[24895]: cluster 2024-09-13T00:35:13.597478+0000 mon.a (mon.0) 874 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2024-09-13T00:35:14.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:14 smithi088 bash[24895]: cluster 2024-09-13T00:35:13.870930+0000 mgr.y (mgr.14397) 217 : cluster [DBG] pgmap v218: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:15.025 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:14 smithi077 bash[32040]: cluster 2024-09-13T00:35:13.597478+0000 mon.a (mon.0) 874 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2024-09-13T00:35:15.025 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:14 smithi077 bash[32040]: cluster 2024-09-13T00:35:13.870930+0000 mgr.y (mgr.14397) 217 : cluster [DBG] pgmap v218: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:15.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:14 smithi077 bash[22596]: cluster 2024-09-13T00:35:13.597478+0000 mon.a (mon.0) 874 : cluster [DBG] osdmap e93: 8 total, 8 up, 8 in 2024-09-13T00:35:15.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:14 smithi077 bash[22596]: cluster 2024-09-13T00:35:13.870930+0000 mgr.y (mgr.14397) 217 : cluster [DBG] pgmap v218: 105 pgs: 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:15.607 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 2 tests from TestMirroringWatcher (2127 ms total) 2024-09-13T00:35:15.607 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:35:15.607 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 57 tests from TestMigration 2024-09-13T00:35:15.685 INFO:teuthology.orchestra.run.smithi077.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T00:35:15.685 INFO:teuthology.orchestra.run.smithi077.stdout:alertmanager.a smithi077 *:9093,9094 running (4m) 25s ago 5m 13.3M - 0.25.0 c8568f914cd2 470a7e35c22c 2024-09-13T00:35:15.685 INFO:teuthology.orchestra.run.smithi077.stdout:grafana.a smithi088 *:3000 running (4m) 24s ago 4m 79.3M - 9.4.7 954c08fa6188 0ce8b83f5ce4 2024-09-13T00:35:15.685 INFO:teuthology.orchestra.run.smithi077.stdout:mds.foo.smithi077.bigbbf smithi077 running (2m) 25s ago 2m 11.7M - 17.2.7-1627-gbbdadfa3 214985049bdb 30b25ab8f2dd 2024-09-13T00:35:15.685 INFO:teuthology.orchestra.run.smithi077.stdout:mds.foo.smithi088.krmowh smithi088 running (2m) 24s ago 2m 13.3M - 17.2.7-1627-gbbdadfa3 214985049bdb ebef6ef73335 2024-09-13T00:35:15.685 INFO:teuthology.orchestra.run.smithi077.stdout:mgr.x smithi088 *:8443 running (13m) 24s ago 13m 389M - 17.2.7-1627-gbbdadfa3 214985049bdb c5737225d5b3 2024-09-13T00:35:15.685 INFO:teuthology.orchestra.run.smithi077.stdout:mgr.y smithi077 *:9283,8765 running (16m) 25s ago 16m 442M - 17.2.7-1627-gbbdadfa3 214985049bdb f857aaa8aef0 2024-09-13T00:35:15.685 INFO:teuthology.orchestra.run.smithi077.stdout:mon.a smithi077 running (16m) 25s ago 16m 50.1M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 893f92df3327 2024-09-13T00:35:15.685 INFO:teuthology.orchestra.run.smithi077.stdout:mon.b smithi088 running (14m) 24s ago 14m 38.8M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb f980dbb0ec7c 2024-09-13T00:35:15.686 INFO:teuthology.orchestra.run.smithi077.stdout:mon.c smithi077 running (14m) 25s ago 14m 38.7M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 875f78edad39 2024-09-13T00:35:15.686 INFO:teuthology.orchestra.run.smithi077.stdout:node-exporter.a smithi077 *:9100 running (5m) 25s ago 5m 8064k - 1.5.0 0da6a335fe13 3a7a77810bc0 2024-09-13T00:35:15.686 INFO:teuthology.orchestra.run.smithi077.stdout:node-exporter.b smithi088 *:9100 running (5m) 24s ago 5m 8275k - 1.5.0 0da6a335fe13 9b46ebe57f2d 2024-09-13T00:35:15.686 INFO:teuthology.orchestra.run.smithi077.stdout:osd.0 smithi077 running (12m) 25s ago 13m 46.4M 4060M 17.2.7-1627-gbbdadfa3 214985049bdb a29e831be9aa 2024-09-13T00:35:15.686 INFO:teuthology.orchestra.run.smithi077.stdout:osd.1 smithi077 running (12m) 25s ago 12m 45.8M 4060M 17.2.7-1627-gbbdadfa3 214985049bdb 8590b3b4d2e9 2024-09-13T00:35:15.686 INFO:teuthology.orchestra.run.smithi077.stdout:osd.2 smithi077 running (11m) 25s ago 11m 43.3M 4060M 17.2.7-1627-gbbdadfa3 214985049bdb b2ecae227e58 2024-09-13T00:35:15.686 INFO:teuthology.orchestra.run.smithi077.stdout:osd.3 smithi077 running (10m) 25s ago 10m 43.5M 4060M 17.2.7-1627-gbbdadfa3 214985049bdb d8c3c25f4ebd 2024-09-13T00:35:15.686 INFO:teuthology.orchestra.run.smithi077.stdout:osd.4 smithi088 running (9m) 24s ago 9m 44.1M 4060M 17.2.7-1627-gbbdadfa3 214985049bdb 2c9ea6919060 2024-09-13T00:35:15.686 INFO:teuthology.orchestra.run.smithi077.stdout:osd.5 smithi088 running (8m) 24s ago 8m 43.5M 4060M 17.2.7-1627-gbbdadfa3 214985049bdb e89a6b298b28 2024-09-13T00:35:15.686 INFO:teuthology.orchestra.run.smithi077.stdout:osd.6 smithi088 running (7m) 24s ago 7m 45.0M 4060M 17.2.7-1627-gbbdadfa3 214985049bdb 599c9665873c 2024-09-13T00:35:15.686 INFO:teuthology.orchestra.run.smithi077.stdout:osd.7 smithi088 running (6m) 24s ago 6m 42.5M 4060M 17.2.7-1627-gbbdadfa3 214985049bdb 0ff0388ffa3c 2024-09-13T00:35:15.686 INFO:teuthology.orchestra.run.smithi077.stdout:prometheus.a smithi088 *:9095 running (4m) 24s ago 5m 48.1M - 2.43.0 a07b618ecd1d 3bbd3b66a6ce 2024-09-13T00:35:15.942 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:15 smithi077 bash[22596]: cluster 2024-09-13T00:35:14.593640+0000 mon.a (mon.0) 875 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:35:15.942 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:15 smithi077 bash[22596]: cluster 2024-09-13T00:35:14.593681+0000 mon.a (mon.0) 876 : cluster [INF] Cluster is now healthy 2024-09-13T00:35:15.942 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:15 smithi077 bash[22596]: cluster 2024-09-13T00:35:14.613593+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2024-09-13T00:35:15.942 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:15 smithi077 bash[32040]: cluster 2024-09-13T00:35:14.593640+0000 mon.a (mon.0) 875 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:35:15.942 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:15 smithi077 bash[32040]: cluster 2024-09-13T00:35:14.593681+0000 mon.a (mon.0) 876 : cluster [INF] Cluster is now healthy 2024-09-13T00:35:15.942 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:15 smithi077 bash[32040]: cluster 2024-09-13T00:35:14.613593+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2024-09-13T00:35:16.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:15 smithi088 bash[24895]: cluster 2024-09-13T00:35:14.593640+0000 mon.a (mon.0) 875 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:35:16.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:15 smithi088 bash[24895]: cluster 2024-09-13T00:35:14.593681+0000 mon.a (mon.0) 876 : cluster [INF] Cluster is now healthy 2024-09-13T00:35:16.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:15 smithi088 bash[24895]: cluster 2024-09-13T00:35:14.613593+0000 mon.a (mon.0) 877 : cluster [DBG] osdmap e94: 8 total, 8 up, 8 in 2024-09-13T00:35:16.943 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:16 smithi077 bash[22596]: cluster 2024-09-13T00:35:15.609400+0000 mon.a (mon.0) 878 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2024-09-13T00:35:16.943 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:16 smithi077 bash[22596]: audit 2024-09-13T00:35:15.677247+0000 mgr.y (mgr.14397) 218 : audit [DBG] from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:35:16.944 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:16 smithi077 bash[22596]: cluster 2024-09-13T00:35:15.871330+0000 mgr.y (mgr.14397) 219 : cluster [DBG] pgmap v221: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:16.944 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:16 smithi077 bash[32040]: cluster 2024-09-13T00:35:15.609400+0000 mon.a (mon.0) 878 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2024-09-13T00:35:16.944 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:16 smithi077 bash[32040]: audit 2024-09-13T00:35:15.677247+0000 mgr.y (mgr.14397) 218 : audit [DBG] from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:35:16.944 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:16 smithi077 bash[32040]: cluster 2024-09-13T00:35:15.871330+0000 mgr.y (mgr.14397) 219 : cluster [DBG] pgmap v221: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:16.945 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'echo wait for minority of mons to upgrade' 2024-09-13T00:35:17.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:16 smithi088 bash[24895]: cluster 2024-09-13T00:35:15.609400+0000 mon.a (mon.0) 878 : cluster [DBG] osdmap e95: 8 total, 8 up, 8 in 2024-09-13T00:35:17.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:16 smithi088 bash[24895]: audit 2024-09-13T00:35:15.677247+0000 mgr.y (mgr.14397) 218 : audit [DBG] from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:35:17.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:16 smithi088 bash[24895]: cluster 2024-09-13T00:35:15.871330+0000 mgr.y (mgr.14397) 219 : cluster [DBG] pgmap v221: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:18.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:18 smithi077 bash[22596]: cluster 2024-09-13T00:35:16.652071+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-09-13T00:35:18.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:18 smithi077 bash[22596]: audit 2024-09-13T00:35:16.655710+0000 mon.c (mon.2) 134 : audit [INF] from='client.? 172.21.15.77:0/2525640017' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:18.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:18 smithi077 bash[22596]: audit 2024-09-13T00:35:16.657798+0000 mon.a (mon.0) 880 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:18.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:18 smithi077 bash[32040]: cluster 2024-09-13T00:35:16.652071+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-09-13T00:35:18.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:18 smithi077 bash[32040]: audit 2024-09-13T00:35:16.655710+0000 mon.c (mon.2) 134 : audit [INF] from='client.? 172.21.15.77:0/2525640017' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:18.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:18 smithi077 bash[32040]: audit 2024-09-13T00:35:16.657798+0000 mon.a (mon.0) 880 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:18.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:18 smithi088 bash[24895]: cluster 2024-09-13T00:35:16.652071+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-09-13T00:35:18.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:18 smithi088 bash[24895]: audit 2024-09-13T00:35:16.655710+0000 mon.c (mon.2) 134 : audit [INF] from='client.? 172.21.15.77:0/2525640017' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:18.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:18 smithi088 bash[24895]: audit 2024-09-13T00:35:16.657798+0000 mon.a (mon.0) 880 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:35:18.648 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.Empty 2024-09-13T00:35:19.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:19 smithi077 bash[22596]: audit 2024-09-13T00:35:17.651768+0000 mon.a (mon.0) 881 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:19.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:19 smithi077 bash[22596]: cluster 2024-09-13T00:35:17.652239+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-09-13T00:35:19.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:19 smithi077 bash[22596]: cluster 2024-09-13T00:35:17.871893+0000 mgr.y (mgr.14397) 220 : cluster [DBG] pgmap v224: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:19.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:19 smithi077 bash[22596]: cluster 2024-09-13T00:35:18.650279+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2024-09-13T00:35:19.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:19 smithi077 bash[32040]: audit 2024-09-13T00:35:17.651768+0000 mon.a (mon.0) 881 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:19.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:19 smithi077 bash[32040]: cluster 2024-09-13T00:35:17.652239+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-09-13T00:35:19.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:19 smithi077 bash[32040]: cluster 2024-09-13T00:35:17.871893+0000 mgr.y (mgr.14397) 220 : cluster [DBG] pgmap v224: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:19.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:19 smithi077 bash[32040]: cluster 2024-09-13T00:35:18.650279+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2024-09-13T00:35:19.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:19 smithi088 bash[24895]: audit 2024-09-13T00:35:17.651768+0000 mon.a (mon.0) 881 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:35:19.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:19 smithi088 bash[24895]: cluster 2024-09-13T00:35:17.652239+0000 mon.a (mon.0) 882 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-09-13T00:35:19.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:19 smithi088 bash[24895]: cluster 2024-09-13T00:35:17.871893+0000 mgr.y (mgr.14397) 220 : cluster [DBG] pgmap v224: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:19.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:19 smithi088 bash[24895]: cluster 2024-09-13T00:35:18.650279+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e98: 8 total, 8 up, 8 in 2024-09-13T00:35:20.735 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:20.755 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:21.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:20 smithi077 bash[32040]: cluster 2024-09-13T00:35:19.659386+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2024-09-13T00:35:21.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:20 smithi077 bash[32040]: cluster 2024-09-13T00:35:19.872641+0000 mgr.y (mgr.14397) 221 : cluster [DBG] pgmap v227: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:21.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:20 smithi077 bash[22596]: cluster 2024-09-13T00:35:19.659386+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2024-09-13T00:35:21.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:20 smithi077 bash[22596]: cluster 2024-09-13T00:35:19.872641+0000 mgr.y (mgr.14397) 221 : cluster [DBG] pgmap v227: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:21.067 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:21.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:35:20 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:35:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:35:21.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:20 smithi088 bash[24895]: cluster 2024-09-13T00:35:19.659386+0000 mon.a (mon.0) 884 : cluster [DBG] osdmap e99: 8 total, 8 up, 8 in 2024-09-13T00:35:21.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:20 smithi088 bash[24895]: cluster 2024-09-13T00:35:19.872641+0000 mgr.y (mgr.14397) 221 : cluster [DBG] pgmap v227: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:21.243 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:35:21.411 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.Empty (2762 ms) 2024-09-13T00:35:21.411 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.OtherName 2024-09-13T00:35:21.482 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:21.508 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:21.754 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:35:21.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:35:21 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:35:21] "GET /metrics HTTP/1.1" 200 242954 "" "Prometheus/2.43.0" 2024-09-13T00:35:21.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:21 smithi077 bash[22596]: cluster 2024-09-13T00:35:20.649805+0000 mon.a (mon.0) 885 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:35:21.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:21 smithi077 bash[22596]: cluster 2024-09-13T00:35:20.666306+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2024-09-13T00:35:21.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:21 smithi077 bash[32040]: cluster 2024-09-13T00:35:20.649805+0000 mon.a (mon.0) 885 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:35:21.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:21 smithi077 bash[32040]: cluster 2024-09-13T00:35:20.666306+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2024-09-13T00:35:21.795 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:21.910 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:35:22.084 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.OtherName (673 ms) 2024-09-13T00:35:22.084 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.OtherPool 2024-09-13T00:35:22.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:21 smithi088 bash[24895]: cluster 2024-09-13T00:35:20.649805+0000 mon.a (mon.0) 885 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:35:22.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:21 smithi088 bash[24895]: cluster 2024-09-13T00:35:20.666306+0000 mon.a (mon.0) 886 : cluster [DBG] osdmap e100: 8 total, 8 up, 8 in 2024-09-13T00:35:22.154 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:22.182 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:23.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:22 smithi077 bash[22596]: cluster 2024-09-13T00:35:21.873183+0000 mgr.y (mgr.14397) 222 : cluster [DBG] pgmap v229: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:23.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:22 smithi077 bash[32040]: cluster 2024-09-13T00:35:21.873183+0000 mgr.y (mgr.14397) 222 : cluster [DBG] pgmap v229: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:23.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:22 smithi088 bash[24895]: cluster 2024-09-13T00:35:21.873183+0000 mgr.y (mgr.14397) 222 : cluster [DBG] pgmap v229: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:35:23.960 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:24.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:23 smithi077 bash[22596]: cluster 2024-09-13T00:35:22.682884+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2024-09-13T00:35:24.025 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:23 smithi077 bash[32040]: cluster 2024-09-13T00:35:22.682884+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2024-09-13T00:35:24.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:23 smithi088 bash[24895]: cluster 2024-09-13T00:35:22.682884+0000 mon.a (mon.0) 887 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2024-09-13T00:35:24.122 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:35:24.278 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.OtherPool (2195 ms) 2024-09-13T00:35:24.279 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.OtherNamespace 2024-09-13T00:35:24.326 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:24.349 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:24.570 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:24.702 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:35:24.835 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.OtherNamespace (556 ms) 2024-09-13T00:35:24.835 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.DataPool 2024-09-13T00:35:24.880 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:24.894 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:24.960 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:24 smithi077 bash[32040]: cluster 2024-09-13T00:35:23.699266+0000 mon.a (mon.0) 888 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-09-13T00:35:24.960 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:24 smithi077 bash[32040]: cluster 2024-09-13T00:35:23.874963+0000 mgr.y (mgr.14397) 223 : cluster [DBG] pgmap v232: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 363 KiB/s rd, 22 KiB/s wr, 461 op/s 2024-09-13T00:35:24.960 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:24 smithi077 bash[22596]: cluster 2024-09-13T00:35:23.699266+0000 mon.a (mon.0) 888 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-09-13T00:35:24.960 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:24 smithi077 bash[22596]: cluster 2024-09-13T00:35:23.874963+0000 mgr.y (mgr.14397) 223 : cluster [DBG] pgmap v232: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 363 KiB/s rd, 22 KiB/s wr, 461 op/s 2024-09-13T00:35:25.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:24 smithi088 bash[24895]: cluster 2024-09-13T00:35:23.699266+0000 mon.a (mon.0) 888 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-09-13T00:35:25.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:24 smithi088 bash[24895]: cluster 2024-09-13T00:35:23.874963+0000 mgr.y (mgr.14397) 223 : cluster [DBG] pgmap v232: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 363 KiB/s rd, 22 KiB/s wr, 461 op/s 2024-09-13T00:35:25.107 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:25.205 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:35:25.307 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.DataPool (472 ms) 2024-09-13T00:35:25.307 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.AbortAfterPrepare 2024-09-13T00:35:25.356 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:25.377 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:25.600 INFO:tasks.workunit.client.0.smithi077.stdout:migration_abort 2024-09-13T00:35:25.757 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.AbortAfterPrepare (451 ms) 2024-09-13T00:35:25.757 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.AbortAfterFailedPrepare 2024-09-13T00:35:25.805 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:25.824 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:25.965 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.AbortAfterFailedPrepare (207 ms) 2024-09-13T00:35:25.965 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.AbortAfterExecute 2024-09-13T00:35:26.012 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:26.030 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:26.269 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:26.382 INFO:tasks.workunit.client.0.smithi077.stdout:migration_abort 2024-09-13T00:35:26.537 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.AbortAfterExecute (572 ms) 2024-09-13T00:35:26.537 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.OtherPoolAbortAfterExecute 2024-09-13T00:35:26.583 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:26.600 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:26.796 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:26.912 INFO:tasks.workunit.client.0.smithi077.stdout:migration_abort 2024-09-13T00:35:27.060 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.OtherPoolAbortAfterExecute (523 ms) 2024-09-13T00:35:27.060 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.OtherNamespaceAbortAfterExecute 2024-09-13T00:35:27.108 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:27.129 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:27.330 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:27.428 INFO:tasks.workunit.client.0.smithi077.stdout:migration_abort 2024-09-13T00:35:27.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:27 smithi077 bash[22596]: cluster 2024-09-13T00:35:25.875428+0000 mgr.y (mgr.14397) 224 : cluster [DBG] pgmap v233: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 KiB/s rd, 16 KiB/s wr, 324 op/s 2024-09-13T00:35:27.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:27 smithi077 bash[22596]: cluster 2024-09-13T00:35:26.109089+0000 mon.a (mon.0) 889 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2024-09-13T00:35:27.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:27 smithi077 bash[32040]: cluster 2024-09-13T00:35:25.875428+0000 mgr.y (mgr.14397) 224 : cluster [DBG] pgmap v233: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 KiB/s rd, 16 KiB/s wr, 324 op/s 2024-09-13T00:35:27.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:27 smithi077 bash[32040]: cluster 2024-09-13T00:35:26.109089+0000 mon.a (mon.0) 889 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2024-09-13T00:35:27.585 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.OtherNamespaceAbortAfterExecute (525 ms) 2024-09-13T00:35:27.585 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.MirroringSamePool 2024-09-13T00:35:27.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:27 smithi088 bash[24895]: cluster 2024-09-13T00:35:25.875428+0000 mgr.y (mgr.14397) 224 : cluster [DBG] pgmap v233: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 KiB/s rd, 16 KiB/s wr, 324 op/s 2024-09-13T00:35:27.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:27 smithi088 bash[24895]: cluster 2024-09-13T00:35:26.109089+0000 mon.a (mon.0) 889 : cluster [DBG] osdmap e103: 8 total, 8 up, 8 in 2024-09-13T00:35:27.637 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:27.655 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:35:27.669 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.MirroringSamePool (84 ms) 2024-09-13T00:35:27.669 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.MirroringAbort 2024-09-13T00:35:27.716 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:27.734 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:35:27.745 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.MirroringAbort (77 ms) 2024-09-13T00:35:27.745 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.MirroringOtherPoolDisabled 2024-09-13T00:35:27.786 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:27.801 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:35:27.811 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.MirroringOtherPoolDisabled (66 ms) 2024-09-13T00:35:27.812 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.MirroringOtherPoolEnabled 2024-09-13T00:35:27.856 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:27.874 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:35:27.886 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.MirroringOtherPoolEnabled (74 ms) 2024-09-13T00:35:27.886 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.MirroringPool 2024-09-13T00:35:27.935 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:27.955 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:35:27.967 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.MirroringPool (82 ms) 2024-09-13T00:35:27.967 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.Group 2024-09-13T00:35:28.018 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:28.051 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:28.296 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:28.424 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:35:28.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:28 smithi077 bash[22596]: audit 2024-09-13T00:35:27.836154+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:28.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:28 smithi077 bash[32040]: audit 2024-09-13T00:35:27.836154+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:28.571 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.Group (603 ms) 2024-09-13T00:35:28.571 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.GroupAbort 2024-09-13T00:35:28.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:28 smithi088 bash[24895]: audit 2024-09-13T00:35:27.836154+0000 mon.c (mon.2) 135 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:28.615 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:28.651 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:28.978 INFO:tasks.workunit.client.0.smithi077.stdout:migration_abort 2024-09-13T00:35:29.158 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.GroupAbort (587 ms) 2024-09-13T00:35:29.158 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.NoSnaps 2024-09-13T00:35:29.204 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:29.218 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:35:29.297 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 16777218~8388609 2024-09-13T00:35:29.363 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:29.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:29 smithi077 bash[22596]: cluster 2024-09-13T00:35:27.875872+0000 mgr.y (mgr.14397) 225 : cluster [DBG] pgmap v235: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 KiB/s rd, 16 KiB/s wr, 324 op/s 2024-09-13T00:35:29.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:29 smithi077 bash[32040]: cluster 2024-09-13T00:35:27.875872+0000 mgr.y (mgr.14397) 225 : cluster [DBG] pgmap v235: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 KiB/s rd, 16 KiB/s wr, 324 op/s 2024-09-13T00:35:29.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:29 smithi088 bash[24895]: cluster 2024-09-13T00:35:27.875872+0000 mgr.y (mgr.14397) 225 : cluster [DBG] pgmap v235: 137 pgs: 137 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 KiB/s rd, 16 KiB/s wr, 324 op/s 2024-09-13T00:35:29.605 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:29.742 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:35:29.886 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.NoSnaps (729 ms) 2024-09-13T00:35:29.887 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.NoSnapsOtherPool 2024-09-13T00:35:29.934 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:29.952 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:35:30.029 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 16777218~8388609 2024-09-13T00:35:30.083 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:35:30.092 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 16777218~8388609 2024-09-13T00:35:30.149 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:30.393 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:30.563 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:35:30.741 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.NoSnapsOtherPool (854 ms) 2024-09-13T00:35:30.741 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.NoSnapsDataPool 2024-09-13T00:35:30.788 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:30.805 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:35:30.874 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 16777218~8388609 2024-09-13T00:35:30.939 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:31.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:35:30 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:35:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:35:31.205 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:31.370 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:35:31.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:31 smithi077 bash[22596]: cluster 2024-09-13T00:35:29.876579+0000 mgr.y (mgr.14397) 226 : cluster [DBG] pgmap v236: 137 pgs: 137 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1018 KiB/s rd, 67 KiB/s wr, 1.31k op/s 2024-09-13T00:35:31.460 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:31 smithi077 bash[32040]: cluster 2024-09-13T00:35:29.876579+0000 mgr.y (mgr.14397) 226 : cluster [DBG] pgmap v236: 137 pgs: 137 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1018 KiB/s rd, 67 KiB/s wr, 1.31k op/s 2024-09-13T00:35:31.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:31 smithi088 bash[24895]: cluster 2024-09-13T00:35:29.876579+0000 mgr.y (mgr.14397) 226 : cluster [DBG] pgmap v236: 137 pgs: 137 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1018 KiB/s rd, 67 KiB/s wr, 1.31k op/s 2024-09-13T00:35:31.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:35:31 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:35:31] "GET /metrics HTTP/1.1" 200 245644 "" "Prometheus/2.43.0" 2024-09-13T00:35:31.993 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.NoSnapsDataPool (1252 ms) 2024-09-13T00:35:31.993 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.NoSnapsShrinkAfterPrepare 2024-09-13T00:35:32.078 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:32.093 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:35:32.163 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 16777218~8388609 2024-09-13T00:35:32.229 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:32.511 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 41943040 2024-09-13T00:35:32.527 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:32.705 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:35:32.867 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.NoSnapsShrinkAfterPrepare (874 ms) 2024-09-13T00:35:32.867 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroBeforePrepare 2024-09-13T00:35:32.913 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:32.931 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:35:32.996 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 16777218~8388609 2024-09-13T00:35:33.063 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 0 2024-09-13T00:35:33.086 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:33.245 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:33.404 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:35:33.524 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroBeforePrepare (658 ms) 2024-09-13T00:35:33.525 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroAfterPrepare 2024-09-13T00:35:33.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:33 smithi077 bash[32040]: cluster 2024-09-13T00:35:31.877311+0000 mgr.y (mgr.14397) 227 : cluster [DBG] pgmap v237: 137 pgs: 137 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 896 KiB/s rd, 59 KiB/s wr, 1.15k op/s 2024-09-13T00:35:33.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:33 smithi077 bash[32040]: audit 2024-09-13T00:35:32.989397+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:33.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:33 smithi077 bash[22596]: cluster 2024-09-13T00:35:31.877311+0000 mgr.y (mgr.14397) 227 : cluster [DBG] pgmap v237: 137 pgs: 137 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 896 KiB/s rd, 59 KiB/s wr, 1.15k op/s 2024-09-13T00:35:33.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:33 smithi077 bash[22596]: audit 2024-09-13T00:35:32.989397+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:33.598 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:33.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:33 smithi088 bash[24895]: cluster 2024-09-13T00:35:31.877311+0000 mgr.y (mgr.14397) 227 : cluster [DBG] pgmap v237: 137 pgs: 137 active+clean; 581 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 896 KiB/s rd, 59 KiB/s wr, 1.15k op/s 2024-09-13T00:35:33.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:33 smithi088 bash[24895]: audit 2024-09-13T00:35:32.989397+0000 mon.a (mon.0) 890 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:33.624 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:35:33.716 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 16777218~8388609 2024-09-13T00:35:33.786 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:34.210 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 0 2024-09-13T00:35:34.242 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:34.420 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:35:34.558 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroAfterPrepare (1033 ms) 2024-09-13T00:35:34.558 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.NoSnapsExpandAfterPrepare 2024-09-13T00:35:34.635 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:34.663 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:35:34.757 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 16777218~8388609 2024-09-13T00:35:34.828 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:35.276 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 167772160 2024-09-13T00:35:35.297 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:35.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:35 smithi077 bash[22596]: cluster 2024-09-13T00:35:33.878986+0000 mgr.y (mgr.14397) 228 : cluster [DBG] pgmap v238: 137 pgs: 137 active+clean; 161 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 29 MiB/s wr, 1.35k op/s 2024-09-13T00:35:35.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:35 smithi077 bash[32040]: cluster 2024-09-13T00:35:33.878986+0000 mgr.y (mgr.14397) 228 : cluster [DBG] pgmap v238: 137 pgs: 137 active+clean; 161 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 29 MiB/s wr, 1.35k op/s 2024-09-13T00:35:35.546 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:35:35.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:35 smithi088 bash[24895]: cluster 2024-09-13T00:35:33.878986+0000 mgr.y (mgr.14397) 228 : cluster [DBG] pgmap v238: 137 pgs: 137 active+clean; 161 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 29 MiB/s wr, 1.35k op/s 2024-09-13T00:35:35.851 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.NoSnapsExpandAfterPrepare (1294 ms) 2024-09-13T00:35:35.852 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.NoSnapsSnapAfterPrepare 2024-09-13T00:35:35.935 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:35.965 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:35:36.064 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 16777218~8388609 2024-09-13T00:35:36.134 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:36.573 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: after_prepare_snap 2024-09-13T00:35:37.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:37 smithi077 bash[22596]: cluster 2024-09-13T00:35:35.879754+0000 mgr.y (mgr.14397) 229 : cluster [DBG] pgmap v239: 137 pgs: 137 active+clean; 161 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 29 MiB/s wr, 1.35k op/s 2024-09-13T00:35:37.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:37 smithi077 bash[32040]: cluster 2024-09-13T00:35:35.879754+0000 mgr.y (mgr.14397) 229 : cluster [DBG] pgmap v239: 137 pgs: 137 active+clean; 161 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 29 MiB/s wr, 1.35k op/s 2024-09-13T00:35:37.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:37 smithi088 bash[24895]: cluster 2024-09-13T00:35:35.879754+0000 mgr.y (mgr.14397) 229 : cluster [DBG] pgmap v239: 137 pgs: 137 active+clean; 161 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 29 MiB/s wr, 1.35k op/s 2024-09-13T00:35:38.223 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 41943040 2024-09-13T00:35:38.268 INFO:tasks.workunit.client.0.smithi077.stdout:write: * 0~1000 2024-09-13T00:35:38.281 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:38.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:38 smithi077 bash[32040]: cluster 2024-09-13T00:35:37.173436+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2024-09-13T00:35:38.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:38 smithi077 bash[22596]: cluster 2024-09-13T00:35:37.173436+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2024-09-13T00:35:38.527 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:35:38.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:38 smithi088 bash[24895]: cluster 2024-09-13T00:35:37.173436+0000 mon.a (mon.0) 891 : cluster [DBG] osdmap e104: 8 total, 8 up, 8 in 2024-09-13T00:35:38.846 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.NoSnapsSnapAfterPrepare (2994 ms) 2024-09-13T00:35:38.846 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.Snaps 2024-09-13T00:35:38.926 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:38.954 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:35:39.070 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:35:39.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:39 smithi077 bash[22596]: cluster 2024-09-13T00:35:37.880277+0000 mgr.y (mgr.14397) 230 : cluster [DBG] pgmap v241: 137 pgs: 137 active+clean; 161 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 29 MiB/s wr, 1.35k op/s 2024-09-13T00:35:39.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:39 smithi077 bash[22596]: cluster 2024-09-13T00:35:38.205649+0000 mon.a (mon.0) 892 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2024-09-13T00:35:39.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:39 smithi077 bash[32040]: cluster 2024-09-13T00:35:37.880277+0000 mgr.y (mgr.14397) 230 : cluster [DBG] pgmap v241: 137 pgs: 137 active+clean; 161 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 29 MiB/s wr, 1.35k op/s 2024-09-13T00:35:39.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:39 smithi077 bash[32040]: cluster 2024-09-13T00:35:38.205649+0000 mon.a (mon.0) 892 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2024-09-13T00:35:39.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:39 smithi088 bash[24895]: cluster 2024-09-13T00:35:37.880277+0000 mgr.y (mgr.14397) 230 : cluster [DBG] pgmap v241: 137 pgs: 137 active+clean; 161 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 29 MiB/s wr, 1.35k op/s 2024-09-13T00:35:39.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:39 smithi088 bash[24895]: cluster 2024-09-13T00:35:38.205649+0000 mon.a (mon.0) 892 : cluster [DBG] osdmap e105: 8 total, 8 up, 8 in 2024-09-13T00:35:40.240 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 8388609~8388609 2024-09-13T00:35:40.279 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 0~1000 2024-09-13T00:35:40.279 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 1010~1000 2024-09-13T00:35:40.279 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap2 2024-09-13T00:35:40.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:40 smithi077 bash[22596]: cluster 2024-09-13T00:35:39.211086+0000 mon.a (mon.0) 893 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2024-09-13T00:35:40.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:40 smithi077 bash[32040]: cluster 2024-09-13T00:35:39.211086+0000 mon.a (mon.0) 893 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2024-09-13T00:35:40.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:40 smithi088 bash[24895]: cluster 2024-09-13T00:35:39.211086+0000 mon.a (mon.0) 893 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2024-09-13T00:35:41.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:35:40 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:35:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:35:41.525 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:35:41 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:35:41] "GET /metrics HTTP/1.1" 200 245644 "" "Prometheus/2.43.0" 2024-09-13T00:35:41.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:41 smithi077 bash[22596]: cluster 2024-09-13T00:35:39.880957+0000 mgr.y (mgr.14397) 231 : cluster [DBG] pgmap v244: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s rd, 16 MiB/s wr, 576 op/s 2024-09-13T00:35:41.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:41 smithi077 bash[22596]: cluster 2024-09-13T00:35:40.222748+0000 mon.a (mon.0) 894 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2024-09-13T00:35:41.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:41 smithi077 bash[32040]: cluster 2024-09-13T00:35:39.880957+0000 mgr.y (mgr.14397) 231 : cluster [DBG] pgmap v244: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s rd, 16 MiB/s wr, 576 op/s 2024-09-13T00:35:41.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:41 smithi077 bash[32040]: cluster 2024-09-13T00:35:40.222748+0000 mon.a (mon.0) 894 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2024-09-13T00:35:41.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:41 smithi088 bash[24895]: cluster 2024-09-13T00:35:39.880957+0000 mgr.y (mgr.14397) 231 : cluster [DBG] pgmap v244: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 37 MiB/s rd, 16 MiB/s wr, 576 op/s 2024-09-13T00:35:41.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:41 smithi088 bash[24895]: cluster 2024-09-13T00:35:40.222748+0000 mon.a (mon.0) 894 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2024-09-13T00:35:42.255 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 8388609~1000 2024-09-13T00:35:42.273 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 16777228~1000 2024-09-13T00:35:42.274 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 167772160 2024-09-13T00:35:42.294 INFO:tasks.workunit.client.0.smithi077.stdout:write: 2 83886079~8388609 2024-09-13T00:35:42.343 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap3 2024-09-13T00:35:42.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:42 smithi077 bash[22596]: cluster 2024-09-13T00:35:41.224234+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2024-09-13T00:35:42.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:42 smithi077 bash[32040]: cluster 2024-09-13T00:35:41.224234+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2024-09-13T00:35:42.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:42 smithi088 bash[24895]: cluster 2024-09-13T00:35:41.224234+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e108: 8 total, 8 up, 8 in 2024-09-13T00:35:42.776 INFO:teuthology.orchestra.run.smithi077.stdout:wait for minority of mons to upgrade 2024-09-13T00:35:43.343 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:43 smithi077 bash[22596]: cluster 2024-09-13T00:35:41.881769+0000 mgr.y (mgr.14397) 232 : cluster [DBG] pgmap v247: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 56 MiB/s rd, 24 MiB/s wr, 864 op/s 2024-09-13T00:35:43.343 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:43 smithi077 bash[22596]: cluster 2024-09-13T00:35:42.244532+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2024-09-13T00:35:43.343 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:43 smithi077 bash[22596]: audit 2024-09-13T00:35:42.836689+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:43.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:43 smithi088 bash[24895]: cluster 2024-09-13T00:35:41.881769+0000 mgr.y (mgr.14397) 232 : cluster [DBG] pgmap v247: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 56 MiB/s rd, 24 MiB/s wr, 864 op/s 2024-09-13T00:35:43.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:43 smithi088 bash[24895]: cluster 2024-09-13T00:35:42.244532+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2024-09-13T00:35:43.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:43 smithi088 bash[24895]: audit 2024-09-13T00:35:42.836689+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:43.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:43 smithi077 bash[32040]: cluster 2024-09-13T00:35:41.881769+0000 mgr.y (mgr.14397) 232 : cluster [DBG] pgmap v247: 137 pgs: 137 active+clean; 193 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 56 MiB/s rd, 24 MiB/s wr, 864 op/s 2024-09-13T00:35:43.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:43 smithi077 bash[32040]: cluster 2024-09-13T00:35:42.244532+0000 mon.a (mon.0) 896 : cluster [DBG] osdmap e109: 8 total, 8 up, 8 in 2024-09-13T00:35:43.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:43 smithi077 bash[32040]: audit 2024-09-13T00:35:42.836689+0000 mon.c (mon.2) 136 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:44.284 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:44.339 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 83886079~1 2024-09-13T00:35:44.359 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:44.407 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-09-13T00:35:44.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:44 smithi077 bash[22596]: cluster 2024-09-13T00:35:43.250733+0000 mon.a (mon.0) 897 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-09-13T00:35:44.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:44 smithi077 bash[32040]: cluster 2024-09-13T00:35:43.250733+0000 mon.a (mon.0) 897 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-09-13T00:35:44.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:44 smithi088 bash[24895]: cluster 2024-09-13T00:35:43.250733+0000 mon.a (mon.0) 897 : cluster [DBG] osdmap e110: 8 total, 8 up, 8 in 2024-09-13T00:35:45.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:45 smithi077 bash[22596]: cluster 2024-09-13T00:35:43.883353+0000 mgr.y (mgr.14397) 233 : cluster [DBG] pgmap v250: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 12 MiB/s wr, 153 op/s 2024-09-13T00:35:45.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:45 smithi077 bash[22596]: cluster 2024-09-13T00:35:44.272205+0000 mon.a (mon.0) 898 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-09-13T00:35:45.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:45 smithi077 bash[22596]: audit 2024-09-13T00:35:44.493803+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:45.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:45 smithi077 bash[22596]: cephadm 2024-09-13T00:35:44.495042+0000 mgr.y (mgr.14397) 234 : cephadm [INF] Upgrade: Target is version 18.2.4-837-g7a853b17 (unknown) 2024-09-13T00:35:45.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:45 smithi077 bash[22596]: cephadm 2024-09-13T00:35:44.495177+0000 mgr.y (mgr.14397) 235 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6'] 2024-09-13T00:35:45.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:45 smithi077 bash[22596]: audit 2024-09-13T00:35:44.497131+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:35:45.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:45 smithi077 bash[22596]: audit 2024-09-13T00:35:44.510436+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:45.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:45 smithi077 bash[22596]: cephadm 2024-09-13T00:35:44.512081+0000 mgr.y (mgr.14397) 236 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T00:35:45.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:45 smithi077 bash[32040]: cluster 2024-09-13T00:35:43.883353+0000 mgr.y (mgr.14397) 233 : cluster [DBG] pgmap v250: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 12 MiB/s wr, 153 op/s 2024-09-13T00:35:45.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:45 smithi077 bash[32040]: cluster 2024-09-13T00:35:44.272205+0000 mon.a (mon.0) 898 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-09-13T00:35:45.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:45 smithi077 bash[32040]: audit 2024-09-13T00:35:44.493803+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:45.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:45 smithi077 bash[32040]: cephadm 2024-09-13T00:35:44.495042+0000 mgr.y (mgr.14397) 234 : cephadm [INF] Upgrade: Target is version 18.2.4-837-g7a853b17 (unknown) 2024-09-13T00:35:45.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:45 smithi077 bash[32040]: cephadm 2024-09-13T00:35:44.495177+0000 mgr.y (mgr.14397) 235 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6'] 2024-09-13T00:35:45.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:45 smithi077 bash[32040]: audit 2024-09-13T00:35:44.497131+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:35:45.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:45 smithi077 bash[32040]: audit 2024-09-13T00:35:44.510436+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:45.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:45 smithi077 bash[32040]: cephadm 2024-09-13T00:35:44.512081+0000 mgr.y (mgr.14397) 236 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T00:35:45.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:45 smithi088 bash[24895]: cluster 2024-09-13T00:35:43.883353+0000 mgr.y (mgr.14397) 233 : cluster [DBG] pgmap v250: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 12 MiB/s wr, 153 op/s 2024-09-13T00:35:45.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:45 smithi088 bash[24895]: cluster 2024-09-13T00:35:44.272205+0000 mon.a (mon.0) 898 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-09-13T00:35:45.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:45 smithi088 bash[24895]: audit 2024-09-13T00:35:44.493803+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:45.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:45 smithi088 bash[24895]: cephadm 2024-09-13T00:35:44.495042+0000 mgr.y (mgr.14397) 234 : cephadm [INF] Upgrade: Target is version 18.2.4-837-g7a853b17 (unknown) 2024-09-13T00:35:45.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:45 smithi088 bash[24895]: cephadm 2024-09-13T00:35:44.495177+0000 mgr.y (mgr.14397) 235 : cephadm [INF] Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6, digests ['quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6'] 2024-09-13T00:35:45.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:45 smithi088 bash[24895]: audit 2024-09-13T00:35:44.497131+0000 mon.c (mon.2) 137 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:35:45.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:45 smithi088 bash[24895]: audit 2024-09-13T00:35:44.510436+0000 mon.a (mon.0) 900 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:35:45.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:45 smithi088 bash[24895]: cephadm 2024-09-13T00:35:44.512081+0000 mgr.y (mgr.14397) 236 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.y) 2024-09-13T00:35:46.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:46 smithi077 bash[22596]: cephadm 2024-09-13T00:35:44.749398+0000 mgr.y (mgr.14397) 237 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 on smithi088 2024-09-13T00:35:46.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:46 smithi077 bash[22596]: cluster 2024-09-13T00:35:45.498119+0000 mon.a (mon.0) 901 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-09-13T00:35:46.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:46 smithi077 bash[32040]: cephadm 2024-09-13T00:35:44.749398+0000 mgr.y (mgr.14397) 237 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 on smithi088 2024-09-13T00:35:46.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:46 smithi077 bash[32040]: cluster 2024-09-13T00:35:45.498119+0000 mon.a (mon.0) 901 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-09-13T00:35:46.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:46 smithi088 bash[24895]: cephadm 2024-09-13T00:35:44.749398+0000 mgr.y (mgr.14397) 237 : cephadm [INF] Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 on smithi088 2024-09-13T00:35:46.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:46 smithi088 bash[24895]: cluster 2024-09-13T00:35:45.498119+0000 mon.a (mon.0) 901 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-09-13T00:35:47.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:47 smithi077 bash[22596]: cluster 2024-09-13T00:35:45.884011+0000 mgr.y (mgr.14397) 238 : cluster [DBG] pgmap v253: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 12 MiB/s wr, 153 op/s 2024-09-13T00:35:47.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:47 smithi077 bash[22596]: cluster 2024-09-13T00:35:46.512848+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2024-09-13T00:35:47.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:47 smithi077 bash[32040]: cluster 2024-09-13T00:35:45.884011+0000 mgr.y (mgr.14397) 238 : cluster [DBG] pgmap v253: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 12 MiB/s wr, 153 op/s 2024-09-13T00:35:47.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:47 smithi077 bash[32040]: cluster 2024-09-13T00:35:46.512848+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2024-09-13T00:35:47.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:47 smithi088 bash[24895]: cluster 2024-09-13T00:35:45.884011+0000 mgr.y (mgr.14397) 238 : cluster [DBG] pgmap v253: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 1.1 MiB/s rd, 12 MiB/s wr, 153 op/s 2024-09-13T00:35:47.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:47 smithi088 bash[24895]: cluster 2024-09-13T00:35:46.512848+0000 mon.a (mon.0) 902 : cluster [DBG] osdmap e113: 8 total, 8 up, 8 in 2024-09-13T00:35:48.398 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:35:48.685 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:35:48.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:48 smithi088 bash[24895]: cluster 2024-09-13T00:35:47.512977+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2024-09-13T00:35:49.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:48 smithi077 bash[22596]: cluster 2024-09-13T00:35:47.512977+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2024-09-13T00:35:49.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:48 smithi077 bash[32040]: cluster 2024-09-13T00:35:47.512977+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e114: 8 total, 8 up, 8 in 2024-09-13T00:35:49.212 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:35:49.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:49 smithi077 bash[22596]: cluster 2024-09-13T00:35:47.884893+0000 mgr.y (mgr.14397) 239 : cluster [DBG] pgmap v256: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:35:49.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:49 smithi077 bash[32040]: cluster 2024-09-13T00:35:47.884893+0000 mgr.y (mgr.14397) 239 : cluster [DBG] pgmap v256: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:35:49.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:49 smithi088 bash[24895]: cluster 2024-09-13T00:35:47.884893+0000 mgr.y (mgr.14397) 239 : cluster [DBG] pgmap v256: 137 pgs: 137 active+clean; 249 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:35:50.829 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:50 smithi077 bash[22596]: cluster 2024-09-13T00:35:49.544898+0000 mon.a (mon.0) 904 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2024-09-13T00:35:50.829 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:50 smithi077 bash[22596]: cluster 2024-09-13T00:35:49.885608+0000 mgr.y (mgr.14397) 240 : cluster [DBG] pgmap v258: 137 pgs: 137 active+clean; 277 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 35 MiB/s rd, 5.5 MiB/s wr, 442 op/s 2024-09-13T00:35:50.830 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:50 smithi077 bash[32040]: cluster 2024-09-13T00:35:49.544898+0000 mon.a (mon.0) 904 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2024-09-13T00:35:50.830 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:50 smithi077 bash[32040]: cluster 2024-09-13T00:35:49.885608+0000 mgr.y (mgr.14397) 240 : cluster [DBG] pgmap v258: 137 pgs: 137 active+clean; 277 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 35 MiB/s rd, 5.5 MiB/s wr, 442 op/s 2024-09-13T00:35:50.850 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:35:50 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:35:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:35:50.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:50 smithi088 bash[24895]: cluster 2024-09-13T00:35:49.544898+0000 mon.a (mon.0) 904 : cluster [DBG] osdmap e115: 8 total, 8 up, 8 in 2024-09-13T00:35:50.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:50 smithi088 bash[24895]: cluster 2024-09-13T00:35:49.885608+0000 mgr.y (mgr.14397) 240 : cluster [DBG] pgmap v258: 137 pgs: 137 active+clean; 277 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 35 MiB/s rd, 5.5 MiB/s wr, 442 op/s 2024-09-13T00:35:51.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:35:51 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:35:51] "GET /metrics HTTP/1.1" 200 246226 "" "Prometheus/2.43.0" 2024-09-13T00:35:51.775 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:51 smithi077 bash[32040]: cluster 2024-09-13T00:35:50.570439+0000 mon.a (mon.0) 905 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2024-09-13T00:35:51.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:51 smithi077 bash[22596]: cluster 2024-09-13T00:35:50.570439+0000 mon.a (mon.0) 905 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2024-09-13T00:35:51.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:51 smithi088 bash[24895]: cluster 2024-09-13T00:35:50.570439+0000 mon.a (mon.0) 905 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2024-09-13T00:35:52.044 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.Snaps (13199 ms) 2024-09-13T00:35:52.045 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.SnapsOtherPool 2024-09-13T00:35:52.126 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:52.155 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:35:52.259 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:35:52.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:52 smithi088 bash[24895]: cluster 2024-09-13T00:35:51.571791+0000 mon.a (mon.0) 906 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2024-09-13T00:35:52.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:52 smithi088 bash[24895]: cluster 2024-09-13T00:35:51.886085+0000 mgr.y (mgr.14397) 241 : cluster [DBG] pgmap v261: 137 pgs: 137 active+clean; 277 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 35 MiB/s rd, 5.5 MiB/s wr, 444 op/s 2024-09-13T00:35:53.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:52 smithi077 bash[22596]: cluster 2024-09-13T00:35:51.571791+0000 mon.a (mon.0) 906 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2024-09-13T00:35:53.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:52 smithi077 bash[22596]: cluster 2024-09-13T00:35:51.886085+0000 mgr.y (mgr.14397) 241 : cluster [DBG] pgmap v261: 137 pgs: 137 active+clean; 277 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 35 MiB/s rd, 5.5 MiB/s wr, 444 op/s 2024-09-13T00:35:53.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:52 smithi077 bash[32040]: cluster 2024-09-13T00:35:51.571791+0000 mon.a (mon.0) 906 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2024-09-13T00:35:53.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:52 smithi077 bash[32040]: cluster 2024-09-13T00:35:51.886085+0000 mgr.y (mgr.14397) 241 : cluster [DBG] pgmap v261: 137 pgs: 137 active+clean; 277 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 35 MiB/s rd, 5.5 MiB/s wr, 444 op/s 2024-09-13T00:35:53.612 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 8388609~8388609 2024-09-13T00:35:53.646 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 0~1000 2024-09-13T00:35:53.646 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 1010~1000 2024-09-13T00:35:53.646 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap2 2024-09-13T00:35:53.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:53 smithi088 bash[24895]: cluster 2024-09-13T00:35:52.595250+0000 mon.a (mon.0) 907 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2024-09-13T00:35:53.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:53 smithi088 bash[24895]: audit 2024-09-13T00:35:53.532935+0000 mon.a (mon.0) 908 : audit [DBG] from='client.? 172.21.15.77:0/427152315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:35:54.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:53 smithi077 bash[22596]: cluster 2024-09-13T00:35:52.595250+0000 mon.a (mon.0) 907 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2024-09-13T00:35:54.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:53 smithi077 bash[22596]: audit 2024-09-13T00:35:53.532935+0000 mon.a (mon.0) 908 : audit [DBG] from='client.? 172.21.15.77:0/427152315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:35:54.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:53 smithi077 bash[32040]: cluster 2024-09-13T00:35:52.595250+0000 mon.a (mon.0) 907 : cluster [DBG] osdmap e118: 8 total, 8 up, 8 in 2024-09-13T00:35:54.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:53 smithi077 bash[32040]: audit 2024-09-13T00:35:53.532935+0000 mon.a (mon.0) 908 : audit [DBG] from='client.? 172.21.15.77:0/427152315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:35:55.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:54 smithi077 bash[22596]: cluster 2024-09-13T00:35:53.599404+0000 mon.a (mon.0) 909 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2024-09-13T00:35:55.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:54 smithi077 bash[22596]: cluster 2024-09-13T00:35:53.887529+0000 mgr.y (mgr.14397) 242 : cluster [DBG] pgmap v264: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 144 op/s 2024-09-13T00:35:55.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:54 smithi077 bash[32040]: cluster 2024-09-13T00:35:53.599404+0000 mon.a (mon.0) 909 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2024-09-13T00:35:55.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:54 smithi077 bash[32040]: cluster 2024-09-13T00:35:53.887529+0000 mgr.y (mgr.14397) 242 : cluster [DBG] pgmap v264: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 144 op/s 2024-09-13T00:35:55.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:54 smithi088 bash[24895]: cluster 2024-09-13T00:35:53.599404+0000 mon.a (mon.0) 909 : cluster [DBG] osdmap e119: 8 total, 8 up, 8 in 2024-09-13T00:35:55.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:54 smithi088 bash[24895]: cluster 2024-09-13T00:35:53.887529+0000 mgr.y (mgr.14397) 242 : cluster [DBG] pgmap v264: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 144 op/s 2024-09-13T00:35:55.636 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 8388609~1000 2024-09-13T00:35:55.649 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 16777228~1000 2024-09-13T00:35:55.649 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 167772160 2024-09-13T00:35:55.669 INFO:tasks.workunit.client.0.smithi077.stdout:write: 2 83886079~8388609 2024-09-13T00:35:55.708 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap3 2024-09-13T00:35:56.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:55 smithi077 bash[22596]: cluster 2024-09-13T00:35:54.611846+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2024-09-13T00:35:56.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:55 smithi077 bash[32040]: cluster 2024-09-13T00:35:54.611846+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2024-09-13T00:35:56.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:55 smithi088 bash[24895]: cluster 2024-09-13T00:35:54.611846+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e120: 8 total, 8 up, 8 in 2024-09-13T00:35:57.025 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:56 smithi077 bash[22596]: cluster 2024-09-13T00:35:55.613826+0000 mon.a (mon.0) 911 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2024-09-13T00:35:57.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:56 smithi077 bash[22596]: cluster 2024-09-13T00:35:55.888215+0000 mgr.y (mgr.14397) 243 : cluster [DBG] pgmap v267: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 144 op/s 2024-09-13T00:35:57.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:56 smithi077 bash[22596]: audit 2024-09-13T00:35:55.896446+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.77:0/2807813526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:35:57.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:56 smithi077 bash[22596]: cluster 2024-09-13T00:35:56.117960+0000 mon.a (mon.0) 912 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2024-09-13T00:35:57.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:56 smithi077 bash[32040]: cluster 2024-09-13T00:35:55.613826+0000 mon.a (mon.0) 911 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2024-09-13T00:35:57.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:56 smithi077 bash[32040]: cluster 2024-09-13T00:35:55.888215+0000 mgr.y (mgr.14397) 243 : cluster [DBG] pgmap v267: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 144 op/s 2024-09-13T00:35:57.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:56 smithi077 bash[32040]: audit 2024-09-13T00:35:55.896446+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.77:0/2807813526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:35:57.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:56 smithi077 bash[32040]: cluster 2024-09-13T00:35:56.117960+0000 mon.a (mon.0) 912 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2024-09-13T00:35:57.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:56 smithi088 bash[24895]: cluster 2024-09-13T00:35:55.613826+0000 mon.a (mon.0) 911 : cluster [DBG] osdmap e121: 8 total, 8 up, 8 in 2024-09-13T00:35:57.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:56 smithi088 bash[24895]: cluster 2024-09-13T00:35:55.888215+0000 mgr.y (mgr.14397) 243 : cluster [DBG] pgmap v267: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 144 op/s 2024-09-13T00:35:57.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:56 smithi088 bash[24895]: audit 2024-09-13T00:35:55.896446+0000 mon.c (mon.2) 138 : audit [DBG] from='client.? 172.21.15.77:0/2807813526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:35:57.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:56 smithi088 bash[24895]: cluster 2024-09-13T00:35:56.117960+0000 mon.a (mon.0) 912 : cluster [DBG] osdmap e122: 8 total, 8 up, 8 in 2024-09-13T00:35:57.142 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:35:57.203 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 83886079~1 2024-09-13T00:35:57.224 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:35:57.256 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 16777218~8388609 2024-09-13T00:35:57.337 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:35:58.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:58 smithi077 bash[32040]: cluster 2024-09-13T00:35:57.119957+0000 mon.a (mon.0) 913 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2024-09-13T00:35:58.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:58 smithi077 bash[32040]: audit 2024-09-13T00:35:57.837289+0000 mon.c (mon.2) 139 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:58.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:58 smithi077 bash[22596]: cluster 2024-09-13T00:35:57.119957+0000 mon.a (mon.0) 913 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2024-09-13T00:35:58.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:58 smithi077 bash[22596]: audit 2024-09-13T00:35:57.837289+0000 mon.c (mon.2) 139 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:58.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:58 smithi088 bash[24895]: cluster 2024-09-13T00:35:57.119957+0000 mon.a (mon.0) 913 : cluster [DBG] osdmap e123: 8 total, 8 up, 8 in 2024-09-13T00:35:58.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:58 smithi088 bash[24895]: audit 2024-09-13T00:35:57.837289+0000 mon.c (mon.2) 139 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:35:59.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:59 smithi077 bash[32040]: cluster 2024-09-13T00:35:57.888981+0000 mgr.y (mgr.14397) 244 : cluster [DBG] pgmap v270: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:35:59.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:59 smithi077 bash[32040]: cluster 2024-09-13T00:35:58.129563+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2024-09-13T00:35:59.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:35:59 smithi077 bash[32040]: audit 2024-09-13T00:35:58.246473+0000 mon.a (mon.0) 915 : audit [DBG] from='client.? 172.21.15.77:0/35335002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:35:59.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:59 smithi077 bash[22596]: cluster 2024-09-13T00:35:57.888981+0000 mgr.y (mgr.14397) 244 : cluster [DBG] pgmap v270: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:35:59.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:59 smithi077 bash[22596]: cluster 2024-09-13T00:35:58.129563+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2024-09-13T00:35:59.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:35:59 smithi077 bash[22596]: audit 2024-09-13T00:35:58.246473+0000 mon.a (mon.0) 915 : audit [DBG] from='client.? 172.21.15.77:0/35335002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:35:59.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:59 smithi088 bash[24895]: cluster 2024-09-13T00:35:57.888981+0000 mgr.y (mgr.14397) 244 : cluster [DBG] pgmap v270: 137 pgs: 137 active+clean; 281 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:35:59.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:59 smithi088 bash[24895]: cluster 2024-09-13T00:35:58.129563+0000 mon.a (mon.0) 914 : cluster [DBG] osdmap e124: 8 total, 8 up, 8 in 2024-09-13T00:35:59.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:35:59 smithi088 bash[24895]: audit 2024-09-13T00:35:58.246473+0000 mon.a (mon.0) 915 : audit [DBG] from='client.? 172.21.15.77:0/35335002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:00.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:00 smithi077 bash[22596]: cluster 2024-09-13T00:35:59.141020+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2024-09-13T00:36:00.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:00 smithi077 bash[32040]: cluster 2024-09-13T00:35:59.141020+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2024-09-13T00:36:00.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:00 smithi088 bash[24895]: cluster 2024-09-13T00:35:59.141020+0000 mon.a (mon.0) 916 : cluster [DBG] osdmap e125: 8 total, 8 up, 8 in 2024-09-13T00:36:00.986 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:36:01.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:00 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:36:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:36:01.285 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:36:01.462 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:01 smithi077 bash[32040]: cluster 2024-09-13T00:35:59.889734+0000 mgr.y (mgr.14397) 245 : cluster [DBG] pgmap v273: 137 pgs: 137 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 134 KiB/s rd, 12 MiB/s wr, 197 op/s 2024-09-13T00:36:01.462 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:01 smithi077 bash[32040]: cluster 2024-09-13T00:36:00.157063+0000 mon.a (mon.0) 917 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2024-09-13T00:36:01.462 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:01 smithi077 bash[32040]: audit 2024-09-13T00:36:00.614652+0000 mon.a (mon.0) 918 : audit [DBG] from='client.? 172.21.15.77:0/1736495430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:01.462 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:01 smithi077 bash[22596]: cluster 2024-09-13T00:35:59.889734+0000 mgr.y (mgr.14397) 245 : cluster [DBG] pgmap v273: 137 pgs: 137 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 134 KiB/s rd, 12 MiB/s wr, 197 op/s 2024-09-13T00:36:01.463 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:01 smithi077 bash[22596]: cluster 2024-09-13T00:36:00.157063+0000 mon.a (mon.0) 917 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2024-09-13T00:36:01.463 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:01 smithi077 bash[22596]: audit 2024-09-13T00:36:00.614652+0000 mon.a (mon.0) 918 : audit [DBG] from='client.? 172.21.15.77:0/1736495430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:01.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:01 smithi088 bash[24895]: cluster 2024-09-13T00:35:59.889734+0000 mgr.y (mgr.14397) 245 : cluster [DBG] pgmap v273: 137 pgs: 137 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 134 KiB/s rd, 12 MiB/s wr, 197 op/s 2024-09-13T00:36:01.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:01 smithi088 bash[24895]: cluster 2024-09-13T00:36:00.157063+0000 mon.a (mon.0) 917 : cluster [DBG] osdmap e126: 8 total, 8 up, 8 in 2024-09-13T00:36:01.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:01 smithi088 bash[24895]: audit 2024-09-13T00:36:00.614652+0000 mon.a (mon.0) 918 : audit [DBG] from='client.? 172.21.15.77:0/1736495430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:01.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:01 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:36:01] "GET /metrics HTTP/1.1" 200 246293 "" "Prometheus/2.43.0" 2024-09-13T00:36:03.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:03 smithi077 bash[22596]: cluster 2024-09-13T00:36:01.890582+0000 mgr.y (mgr.14397) 246 : cluster [DBG] pgmap v275: 137 pgs: 137 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 113 KiB/s rd, 10 MiB/s wr, 165 op/s 2024-09-13T00:36:03.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:03 smithi077 bash[22596]: cluster 2024-09-13T00:36:02.165570+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2024-09-13T00:36:03.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:03 smithi077 bash[22596]: audit 2024-09-13T00:36:02.967070+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.77:0/2518574085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:03.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:03 smithi077 bash[32040]: cluster 2024-09-13T00:36:01.890582+0000 mgr.y (mgr.14397) 246 : cluster [DBG] pgmap v275: 137 pgs: 137 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 113 KiB/s rd, 10 MiB/s wr, 165 op/s 2024-09-13T00:36:03.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:03 smithi077 bash[32040]: cluster 2024-09-13T00:36:02.165570+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2024-09-13T00:36:03.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:03 smithi077 bash[32040]: audit 2024-09-13T00:36:02.967070+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.77:0/2518574085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:03.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:03 smithi088 bash[24895]: cluster 2024-09-13T00:36:01.890582+0000 mgr.y (mgr.14397) 246 : cluster [DBG] pgmap v275: 137 pgs: 137 active+clean; 337 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 113 KiB/s rd, 10 MiB/s wr, 165 op/s 2024-09-13T00:36:03.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:03 smithi088 bash[24895]: cluster 2024-09-13T00:36:02.165570+0000 mon.a (mon.0) 919 : cluster [DBG] osdmap e127: 8 total, 8 up, 8 in 2024-09-13T00:36:03.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:03 smithi088 bash[24895]: audit 2024-09-13T00:36:02.967070+0000 mon.a (mon.0) 920 : audit [DBG] from='client.? 172.21.15.77:0/2518574085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:04.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:04 smithi077 bash[32040]: cluster 2024-09-13T00:36:03.178817+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2024-09-13T00:36:04.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:04 smithi077 bash[22596]: cluster 2024-09-13T00:36:03.178817+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2024-09-13T00:36:04.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:04 smithi088 bash[24895]: cluster 2024-09-13T00:36:03.178817+0000 mon.a (mon.0) 921 : cluster [DBG] osdmap e128: 8 total, 8 up, 8 in 2024-09-13T00:36:04.760 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.SnapsOtherPool (12715 ms) 2024-09-13T00:36:04.761 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.SnapsDataPool 2024-09-13T00:36:04.834 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:36:04.859 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:36:04.960 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:36:05.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:05 smithi077 bash[22596]: cluster 2024-09-13T00:36:03.892125+0000 mgr.y (mgr.14397) 247 : cluster [DBG] pgmap v278: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 8.4 MiB/s wr, 319 op/s 2024-09-13T00:36:05.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:05 smithi077 bash[22596]: cluster 2024-09-13T00:36:04.199716+0000 mon.a (mon.0) 922 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2024-09-13T00:36:05.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:05 smithi077 bash[32040]: cluster 2024-09-13T00:36:03.892125+0000 mgr.y (mgr.14397) 247 : cluster [DBG] pgmap v278: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 8.4 MiB/s wr, 319 op/s 2024-09-13T00:36:05.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:05 smithi077 bash[32040]: cluster 2024-09-13T00:36:04.199716+0000 mon.a (mon.0) 922 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2024-09-13T00:36:05.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:05 smithi088 bash[24895]: cluster 2024-09-13T00:36:03.892125+0000 mgr.y (mgr.14397) 247 : cluster [DBG] pgmap v278: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 39 MiB/s rd, 8.4 MiB/s wr, 319 op/s 2024-09-13T00:36:05.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:05 smithi088 bash[24895]: cluster 2024-09-13T00:36:04.199716+0000 mon.a (mon.0) 922 : cluster [DBG] osdmap e129: 8 total, 8 up, 8 in 2024-09-13T00:36:06.140 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 8388609~8388609 2024-09-13T00:36:06.177 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 0~1000 2024-09-13T00:36:06.177 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 1010~1000 2024-09-13T00:36:06.177 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap2 2024-09-13T00:36:06.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:06 smithi077 bash[22596]: cluster 2024-09-13T00:36:05.200037+0000 mon.a (mon.0) 923 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2024-09-13T00:36:06.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:06 smithi077 bash[22596]: audit 2024-09-13T00:36:05.322635+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.77:0/1308522563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:06.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:06 smithi077 bash[22596]: cluster 2024-09-13T00:36:06.122789+0000 mon.a (mon.0) 924 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2024-09-13T00:36:06.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:06 smithi077 bash[32040]: cluster 2024-09-13T00:36:05.200037+0000 mon.a (mon.0) 923 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2024-09-13T00:36:06.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:06 smithi077 bash[32040]: audit 2024-09-13T00:36:05.322635+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.77:0/1308522563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:06.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:06 smithi077 bash[32040]: cluster 2024-09-13T00:36:06.122789+0000 mon.a (mon.0) 924 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2024-09-13T00:36:06.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:06 smithi088 bash[24895]: cluster 2024-09-13T00:36:05.200037+0000 mon.a (mon.0) 923 : cluster [DBG] osdmap e130: 8 total, 8 up, 8 in 2024-09-13T00:36:06.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:06 smithi088 bash[24895]: audit 2024-09-13T00:36:05.322635+0000 mon.c (mon.2) 140 : audit [DBG] from='client.? 172.21.15.77:0/1308522563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:06.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:06 smithi088 bash[24895]: cluster 2024-09-13T00:36:06.122789+0000 mon.a (mon.0) 924 : cluster [DBG] osdmap e131: 8 total, 8 up, 8 in 2024-09-13T00:36:07.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:07 smithi077 bash[32040]: cluster 2024-09-13T00:36:05.892934+0000 mgr.y (mgr.14397) 248 : cluster [DBG] pgmap v281: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 379 op/s 2024-09-13T00:36:07.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:07 smithi077 bash[22596]: cluster 2024-09-13T00:36:05.892934+0000 mgr.y (mgr.14397) 248 : cluster [DBG] pgmap v281: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 379 op/s 2024-09-13T00:36:07.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:07 smithi088 bash[24895]: cluster 2024-09-13T00:36:05.892934+0000 mgr.y (mgr.14397) 248 : cluster [DBG] pgmap v281: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 379 op/s 2024-09-13T00:36:08.253 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 8388609~1000 2024-09-13T00:36:08.267 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 16777228~1000 2024-09-13T00:36:08.267 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 167772160 2024-09-13T00:36:08.285 INFO:tasks.workunit.client.0.smithi077.stdout:write: 2 83886079~8388609 2024-09-13T00:36:08.330 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap3 2024-09-13T00:36:08.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:08 smithi077 bash[22596]: cluster 2024-09-13T00:36:07.221062+0000 mon.a (mon.0) 925 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2024-09-13T00:36:08.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:08 smithi077 bash[22596]: audit 2024-09-13T00:36:07.672410+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.77:0/2850000661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:08.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:08 smithi077 bash[32040]: cluster 2024-09-13T00:36:07.221062+0000 mon.a (mon.0) 925 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2024-09-13T00:36:08.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:08 smithi077 bash[32040]: audit 2024-09-13T00:36:07.672410+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.77:0/2850000661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:08.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:08 smithi088 bash[24895]: cluster 2024-09-13T00:36:07.221062+0000 mon.a (mon.0) 925 : cluster [DBG] osdmap e132: 8 total, 8 up, 8 in 2024-09-13T00:36:08.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:08 smithi088 bash[24895]: audit 2024-09-13T00:36:07.672410+0000 mon.a (mon.0) 926 : audit [DBG] from='client.? 172.21.15.77:0/2850000661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:09.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:09 smithi077 bash[22596]: cluster 2024-09-13T00:36:07.893722+0000 mgr.y (mgr.14397) 249 : cluster [DBG] pgmap v284: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:36:09.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:09 smithi077 bash[22596]: cluster 2024-09-13T00:36:08.241164+0000 mon.a (mon.0) 927 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2024-09-13T00:36:09.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:09 smithi077 bash[32040]: cluster 2024-09-13T00:36:07.893722+0000 mgr.y (mgr.14397) 249 : cluster [DBG] pgmap v284: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:36:09.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:09 smithi077 bash[32040]: cluster 2024-09-13T00:36:08.241164+0000 mon.a (mon.0) 927 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2024-09-13T00:36:09.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:09 smithi088 bash[24895]: cluster 2024-09-13T00:36:07.893722+0000 mgr.y (mgr.14397) 249 : cluster [DBG] pgmap v284: 137 pgs: 137 active+clean; 373 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:36:09.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:09 smithi088 bash[24895]: cluster 2024-09-13T00:36:08.241164+0000 mon.a (mon.0) 927 : cluster [DBG] osdmap e133: 8 total, 8 up, 8 in 2024-09-13T00:36:10.272 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:36:10.324 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 83886079~1 2024-09-13T00:36:10.344 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:36:10.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:10 smithi077 bash[22596]: cluster 2024-09-13T00:36:09.248524+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2024-09-13T00:36:10.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:10 smithi077 bash[22596]: audit 2024-09-13T00:36:10.019854+0000 mon.a (mon.0) 929 : audit [DBG] from='client.? 172.21.15.77:0/2727329784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:10.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:10 smithi077 bash[32040]: cluster 2024-09-13T00:36:09.248524+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2024-09-13T00:36:10.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:10 smithi077 bash[32040]: audit 2024-09-13T00:36:10.019854+0000 mon.a (mon.0) 929 : audit [DBG] from='client.? 172.21.15.77:0/2727329784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:10.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:10 smithi088 bash[24895]: cluster 2024-09-13T00:36:09.248524+0000 mon.a (mon.0) 928 : cluster [DBG] osdmap e134: 8 total, 8 up, 8 in 2024-09-13T00:36:10.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:10 smithi088 bash[24895]: audit 2024-09-13T00:36:10.019854+0000 mon.a (mon.0) 929 : audit [DBG] from='client.? 172.21.15.77:0/2727329784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:11.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:10 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:36:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:36:11.525 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:11 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:36:11] "GET /metrics HTTP/1.1" 200 246293 "" "Prometheus/2.43.0" 2024-09-13T00:36:11.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:11 smithi077 bash[22596]: cluster 2024-09-13T00:36:09.894417+0000 mgr.y (mgr.14397) 250 : cluster [DBG] pgmap v287: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 183 op/s 2024-09-13T00:36:11.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:11 smithi077 bash[22596]: cluster 2024-09-13T00:36:10.261044+0000 mon.a (mon.0) 930 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2024-09-13T00:36:11.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:11 smithi077 bash[22596]: cluster 2024-09-13T00:36:11.124795+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2024-09-13T00:36:11.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:11 smithi077 bash[32040]: cluster 2024-09-13T00:36:09.894417+0000 mgr.y (mgr.14397) 250 : cluster [DBG] pgmap v287: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 183 op/s 2024-09-13T00:36:11.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:11 smithi077 bash[32040]: cluster 2024-09-13T00:36:10.261044+0000 mon.a (mon.0) 930 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2024-09-13T00:36:11.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:11 smithi077 bash[32040]: cluster 2024-09-13T00:36:11.124795+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2024-09-13T00:36:11.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:11 smithi088 bash[24895]: cluster 2024-09-13T00:36:09.894417+0000 mgr.y (mgr.14397) 250 : cluster [DBG] pgmap v287: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 183 op/s 2024-09-13T00:36:11.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:11 smithi088 bash[24895]: cluster 2024-09-13T00:36:10.261044+0000 mon.a (mon.0) 930 : cluster [DBG] osdmap e135: 8 total, 8 up, 8 in 2024-09-13T00:36:11.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:11 smithi088 bash[24895]: cluster 2024-09-13T00:36:11.124795+0000 mon.a (mon.0) 931 : cluster [DBG] osdmap e136: 8 total, 8 up, 8 in 2024-09-13T00:36:13.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:13 smithi077 bash[22596]: cluster 2024-09-13T00:36:11.895182+0000 mgr.y (mgr.14397) 251 : cluster [DBG] pgmap v290: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 183 op/s 2024-09-13T00:36:13.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:13 smithi077 bash[22596]: cluster 2024-09-13T00:36:12.122642+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2024-09-13T00:36:13.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:13 smithi077 bash[22596]: audit 2024-09-13T00:36:12.373834+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.77:0/1488817821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:13.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:13 smithi077 bash[22596]: audit 2024-09-13T00:36:12.837496+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:13.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:13 smithi077 bash[32040]: cluster 2024-09-13T00:36:11.895182+0000 mgr.y (mgr.14397) 251 : cluster [DBG] pgmap v290: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 183 op/s 2024-09-13T00:36:13.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:13 smithi077 bash[32040]: cluster 2024-09-13T00:36:12.122642+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2024-09-13T00:36:13.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:13 smithi077 bash[32040]: audit 2024-09-13T00:36:12.373834+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.77:0/1488817821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:13.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:13 smithi077 bash[32040]: audit 2024-09-13T00:36:12.837496+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:13.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:13 smithi088 bash[24895]: cluster 2024-09-13T00:36:11.895182+0000 mgr.y (mgr.14397) 251 : cluster [DBG] pgmap v290: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 183 op/s 2024-09-13T00:36:13.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:13 smithi088 bash[24895]: cluster 2024-09-13T00:36:12.122642+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e137: 8 total, 8 up, 8 in 2024-09-13T00:36:13.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:13 smithi088 bash[24895]: audit 2024-09-13T00:36:12.373834+0000 mon.c (mon.2) 141 : audit [DBG] from='client.? 172.21.15.77:0/1488817821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:13.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:13 smithi088 bash[24895]: audit 2024-09-13T00:36:12.837496+0000 mon.c (mon.2) 142 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:13.952 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:36:14.214 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:36:14.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:14 smithi077 bash[22596]: cluster 2024-09-13T00:36:13.136446+0000 mon.a (mon.0) 933 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2024-09-13T00:36:14.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:14 smithi077 bash[32040]: cluster 2024-09-13T00:36:13.136446+0000 mon.a (mon.0) 933 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2024-09-13T00:36:14.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:14 smithi088 bash[24895]: cluster 2024-09-13T00:36:13.136446+0000 mon.a (mon.0) 933 : cluster [DBG] osdmap e138: 8 total, 8 up, 8 in 2024-09-13T00:36:15.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:15 smithi077 bash[22596]: cluster 2024-09-13T00:36:13.897024+0000 mgr.y (mgr.14397) 252 : cluster [DBG] pgmap v293: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 224 op/s 2024-09-13T00:36:15.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:15 smithi077 bash[22596]: audit 2024-09-13T00:36:14.721751+0000 mon.a (mon.0) 934 : audit [DBG] from='client.? 172.21.15.77:0/3156917833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:15.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:15 smithi077 bash[32040]: cluster 2024-09-13T00:36:13.897024+0000 mgr.y (mgr.14397) 252 : cluster [DBG] pgmap v293: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 224 op/s 2024-09-13T00:36:15.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:15 smithi077 bash[32040]: audit 2024-09-13T00:36:14.721751+0000 mon.a (mon.0) 934 : audit [DBG] from='client.? 172.21.15.77:0/3156917833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:15.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:15 smithi088 bash[24895]: cluster 2024-09-13T00:36:13.897024+0000 mgr.y (mgr.14397) 252 : cluster [DBG] pgmap v293: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 224 op/s 2024-09-13T00:36:15.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:15 smithi088 bash[24895]: audit 2024-09-13T00:36:14.721751+0000 mon.a (mon.0) 934 : audit [DBG] from='client.? 172.21.15.77:0/3156917833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:16.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:16 smithi077 bash[22596]: cluster 2024-09-13T00:36:15.169703+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2024-09-13T00:36:16.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:16 smithi077 bash[32040]: cluster 2024-09-13T00:36:15.169703+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2024-09-13T00:36:16.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:16 smithi088 bash[24895]: cluster 2024-09-13T00:36:15.169703+0000 mon.a (mon.0) 935 : cluster [DBG] osdmap e139: 8 total, 8 up, 8 in 2024-09-13T00:36:17.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:17 smithi077 bash[22596]: cluster 2024-09-13T00:36:15.897908+0000 mgr.y (mgr.14397) 253 : cluster [DBG] pgmap v295: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 KiB/s wr, 188 op/s 2024-09-13T00:36:17.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:17 smithi077 bash[22596]: cluster 2024-09-13T00:36:16.179275+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2024-09-13T00:36:17.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:17 smithi077 bash[22596]: audit 2024-09-13T00:36:17.074529+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.77:0/761092624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:17.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:17 smithi077 bash[32040]: cluster 2024-09-13T00:36:15.897908+0000 mgr.y (mgr.14397) 253 : cluster [DBG] pgmap v295: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 KiB/s wr, 188 op/s 2024-09-13T00:36:17.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:17 smithi077 bash[32040]: cluster 2024-09-13T00:36:16.179275+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2024-09-13T00:36:17.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:17 smithi077 bash[32040]: audit 2024-09-13T00:36:17.074529+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.77:0/761092624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:17.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:17 smithi088 bash[24895]: cluster 2024-09-13T00:36:15.897908+0000 mgr.y (mgr.14397) 253 : cluster [DBG] pgmap v295: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 KiB/s wr, 188 op/s 2024-09-13T00:36:17.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:17 smithi088 bash[24895]: cluster 2024-09-13T00:36:16.179275+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e140: 8 total, 8 up, 8 in 2024-09-13T00:36:17.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:17 smithi088 bash[24895]: audit 2024-09-13T00:36:17.074529+0000 mon.c (mon.2) 143 : audit [DBG] from='client.? 172.21.15.77:0/761092624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:17.709 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.SnapsDataPool (12949 ms) 2024-09-13T00:36:17.710 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.SnapsShrinkAfterPrepare 2024-09-13T00:36:17.776 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:36:17.803 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:36:17.908 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:36:18.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:18 smithi077 bash[22596]: cluster 2024-09-13T00:36:17.175612+0000 mon.a (mon.0) 937 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2024-09-13T00:36:18.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:18 smithi077 bash[32040]: cluster 2024-09-13T00:36:17.175612+0000 mon.a (mon.0) 937 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2024-09-13T00:36:18.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:18 smithi088 bash[24895]: cluster 2024-09-13T00:36:17.175612+0000 mon.a (mon.0) 937 : cluster [DBG] osdmap e141: 8 total, 8 up, 8 in 2024-09-13T00:36:19.209 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 8388609~8388609 2024-09-13T00:36:19.237 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 0~1000 2024-09-13T00:36:19.237 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 1010~1000 2024-09-13T00:36:19.237 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap2 2024-09-13T00:36:19.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:19 smithi077 bash[22596]: cluster 2024-09-13T00:36:17.898672+0000 mgr.y (mgr.14397) 254 : cluster [DBG] pgmap v298: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.9 KiB/s wr, 188 op/s 2024-09-13T00:36:19.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:19 smithi077 bash[22596]: cluster 2024-09-13T00:36:18.201387+0000 mon.a (mon.0) 938 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2024-09-13T00:36:19.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:19 smithi077 bash[32040]: cluster 2024-09-13T00:36:17.898672+0000 mgr.y (mgr.14397) 254 : cluster [DBG] pgmap v298: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.9 KiB/s wr, 188 op/s 2024-09-13T00:36:19.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:19 smithi077 bash[32040]: cluster 2024-09-13T00:36:18.201387+0000 mon.a (mon.0) 938 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2024-09-13T00:36:19.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:19 smithi088 bash[24895]: cluster 2024-09-13T00:36:17.898672+0000 mgr.y (mgr.14397) 254 : cluster [DBG] pgmap v298: 137 pgs: 137 active+clean; 393 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 27 MiB/s rd, 9.9 KiB/s wr, 188 op/s 2024-09-13T00:36:19.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:19 smithi088 bash[24895]: cluster 2024-09-13T00:36:18.201387+0000 mon.a (mon.0) 938 : cluster [DBG] osdmap e142: 8 total, 8 up, 8 in 2024-09-13T00:36:20.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:20 smithi077 bash[22596]: cluster 2024-09-13T00:36:19.202731+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2024-09-13T00:36:20.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:20 smithi077 bash[22596]: audit 2024-09-13T00:36:19.440725+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.77:0/2976559318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:20.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:20 smithi077 bash[32040]: cluster 2024-09-13T00:36:19.202731+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2024-09-13T00:36:20.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:20 smithi077 bash[32040]: audit 2024-09-13T00:36:19.440725+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.77:0/2976559318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:20.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:20 smithi088 bash[24895]: cluster 2024-09-13T00:36:19.202731+0000 mon.a (mon.0) 939 : cluster [DBG] osdmap e143: 8 total, 8 up, 8 in 2024-09-13T00:36:20.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:20 smithi088 bash[24895]: audit 2024-09-13T00:36:19.440725+0000 mon.c (mon.2) 144 : audit [DBG] from='client.? 172.21.15.77:0/2976559318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:21.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:20 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:36:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:36:21.150 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 8388609~1000 2024-09-13T00:36:21.178 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 16777228~1000 2024-09-13T00:36:21.179 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 167772160 2024-09-13T00:36:21.198 INFO:tasks.workunit.client.0.smithi077.stdout:write: 2 83886079~8388609 2024-09-13T00:36:21.247 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap3 2024-09-13T00:36:21.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:21 smithi077 bash[22596]: cluster 2024-09-13T00:36:19.899438+0000 mgr.y (mgr.14397) 255 : cluster [DBG] pgmap v301: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-09-13T00:36:21.462 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:21 smithi077 bash[22596]: cluster 2024-09-13T00:36:20.224752+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2024-09-13T00:36:21.462 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:21 smithi077 bash[22596]: cluster 2024-09-13T00:36:21.128753+0000 mon.a (mon.0) 941 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2024-09-13T00:36:21.462 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:21 smithi077 bash[32040]: cluster 2024-09-13T00:36:19.899438+0000 mgr.y (mgr.14397) 255 : cluster [DBG] pgmap v301: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-09-13T00:36:21.462 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:21 smithi077 bash[32040]: cluster 2024-09-13T00:36:20.224752+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2024-09-13T00:36:21.462 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:21 smithi077 bash[32040]: cluster 2024-09-13T00:36:21.128753+0000 mon.a (mon.0) 941 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2024-09-13T00:36:21.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:21 smithi088 bash[24895]: cluster 2024-09-13T00:36:19.899438+0000 mgr.y (mgr.14397) 255 : cluster [DBG] pgmap v301: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-09-13T00:36:21.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:21 smithi088 bash[24895]: cluster 2024-09-13T00:36:20.224752+0000 mon.a (mon.0) 940 : cluster [DBG] osdmap e144: 8 total, 8 up, 8 in 2024-09-13T00:36:21.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:21 smithi088 bash[24895]: cluster 2024-09-13T00:36:21.128753+0000 mon.a (mon.0) 941 : cluster [DBG] osdmap e145: 8 total, 8 up, 8 in 2024-09-13T00:36:21.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:21 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:36:21] "GET /metrics HTTP/1.1" 200 246371 "" "Prometheus/2.43.0" 2024-09-13T00:36:22.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:22 smithi077 bash[22596]: audit 2024-09-13T00:36:21.790716+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.77:0/1613640475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:22.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:22 smithi077 bash[32040]: audit 2024-09-13T00:36:21.790716+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.77:0/1613640475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:22.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:22 smithi088 bash[24895]: audit 2024-09-13T00:36:21.790716+0000 mon.c (mon.2) 145 : audit [DBG] from='client.? 172.21.15.77:0/1613640475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:23.255 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:36:23.305 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 83886079~1 2024-09-13T00:36:23.323 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:36:23.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:23 smithi077 bash[22596]: cluster 2024-09-13T00:36:21.900251+0000 mgr.y (mgr.14397) 256 : cluster [DBG] pgmap v304: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-09-13T00:36:23.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:23 smithi077 bash[22596]: cluster 2024-09-13T00:36:22.231507+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2024-09-13T00:36:23.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:23 smithi077 bash[32040]: cluster 2024-09-13T00:36:21.900251+0000 mgr.y (mgr.14397) 256 : cluster [DBG] pgmap v304: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-09-13T00:36:23.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:23 smithi077 bash[32040]: cluster 2024-09-13T00:36:22.231507+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2024-09-13T00:36:23.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:23 smithi088 bash[24895]: cluster 2024-09-13T00:36:21.900251+0000 mgr.y (mgr.14397) 256 : cluster [DBG] pgmap v304: 137 pgs: 137 active+clean; 409 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 372 op/s 2024-09-13T00:36:23.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:23 smithi088 bash[24895]: cluster 2024-09-13T00:36:22.231507+0000 mon.a (mon.0) 942 : cluster [DBG] osdmap e146: 8 total, 8 up, 8 in 2024-09-13T00:36:24.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:24 smithi077 bash[22596]: cluster 2024-09-13T00:36:23.239605+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2024-09-13T00:36:24.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:24 smithi077 bash[22596]: audit 2024-09-13T00:36:24.138187+0000 mon.a (mon.0) 944 : audit [DBG] from='client.? 172.21.15.77:0/1784603269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:24.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:24 smithi077 bash[32040]: cluster 2024-09-13T00:36:23.239605+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2024-09-13T00:36:24.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:24 smithi077 bash[32040]: audit 2024-09-13T00:36:24.138187+0000 mon.a (mon.0) 944 : audit [DBG] from='client.? 172.21.15.77:0/1784603269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:24.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:24 smithi088 bash[24895]: cluster 2024-09-13T00:36:23.239605+0000 mon.a (mon.0) 943 : cluster [DBG] osdmap e147: 8 total, 8 up, 8 in 2024-09-13T00:36:24.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:24 smithi088 bash[24895]: audit 2024-09-13T00:36:24.138187+0000 mon.a (mon.0) 944 : audit [DBG] from='client.? 172.21.15.77:0/1784603269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:25.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:25 smithi077 bash[22596]: cluster 2024-09-13T00:36:23.901075+0000 mgr.y (mgr.14397) 257 : cluster [DBG] pgmap v307: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-13T00:36:25.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:25 smithi077 bash[22596]: cluster 2024-09-13T00:36:24.252164+0000 mon.a (mon.0) 945 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2024-09-13T00:36:25.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:25 smithi077 bash[32040]: cluster 2024-09-13T00:36:23.901075+0000 mgr.y (mgr.14397) 257 : cluster [DBG] pgmap v307: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-13T00:36:25.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:25 smithi077 bash[32040]: cluster 2024-09-13T00:36:24.252164+0000 mon.a (mon.0) 945 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2024-09-13T00:36:25.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:25 smithi088 bash[24895]: cluster 2024-09-13T00:36:23.901075+0000 mgr.y (mgr.14397) 257 : cluster [DBG] pgmap v307: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-13T00:36:25.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:25 smithi088 bash[24895]: cluster 2024-09-13T00:36:24.252164+0000 mon.a (mon.0) 945 : cluster [DBG] osdmap e148: 8 total, 8 up, 8 in 2024-09-13T00:36:26.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:26 smithi077 bash[22596]: cluster 2024-09-13T00:36:25.253942+0000 mon.a (mon.0) 946 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2024-09-13T00:36:26.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:26 smithi077 bash[32040]: cluster 2024-09-13T00:36:25.253942+0000 mon.a (mon.0) 946 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2024-09-13T00:36:26.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:26 smithi088 bash[24895]: cluster 2024-09-13T00:36:25.253942+0000 mon.a (mon.0) 946 : cluster [DBG] osdmap e149: 8 total, 8 up, 8 in 2024-09-13T00:36:27.063 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 41943040 2024-09-13T00:36:27.119 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:36:27.379 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:36:27.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:27 smithi077 bash[22596]: cluster 2024-09-13T00:36:25.901789+0000 mgr.y (mgr.14397) 258 : cluster [DBG] pgmap v310: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-13T00:36:27.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:27 smithi077 bash[22596]: cluster 2024-09-13T00:36:26.271051+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2024-09-13T00:36:27.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:27 smithi077 bash[22596]: audit 2024-09-13T00:36:26.503096+0000 mon.a (mon.0) 948 : audit [DBG] from='client.? 172.21.15.77:0/3291220848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:27.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:27 smithi077 bash[32040]: cluster 2024-09-13T00:36:25.901789+0000 mgr.y (mgr.14397) 258 : cluster [DBG] pgmap v310: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-13T00:36:27.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:27 smithi077 bash[32040]: cluster 2024-09-13T00:36:26.271051+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2024-09-13T00:36:27.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:27 smithi077 bash[32040]: audit 2024-09-13T00:36:26.503096+0000 mon.a (mon.0) 948 : audit [DBG] from='client.? 172.21.15.77:0/3291220848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:27.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:27 smithi088 bash[24895]: cluster 2024-09-13T00:36:25.901789+0000 mgr.y (mgr.14397) 258 : cluster [DBG] pgmap v310: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-13T00:36:27.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:27 smithi088 bash[24895]: cluster 2024-09-13T00:36:26.271051+0000 mon.a (mon.0) 947 : cluster [DBG] osdmap e150: 8 total, 8 up, 8 in 2024-09-13T00:36:27.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:27 smithi088 bash[24895]: audit 2024-09-13T00:36:26.503096+0000 mon.a (mon.0) 948 : audit [DBG] from='client.? 172.21.15.77:0/3291220848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:28.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:28 smithi077 bash[32040]: audit 2024-09-13T00:36:27.838760+0000 mon.c (mon.2) 146 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:28.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:28 smithi077 bash[22596]: audit 2024-09-13T00:36:27.838760+0000 mon.c (mon.2) 146 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:28.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:28 smithi088 bash[24895]: audit 2024-09-13T00:36:27.838760+0000 mon.c (mon.2) 146 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:29.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:29 smithi077 bash[22596]: cluster 2024-09-13T00:36:27.902430+0000 mgr.y (mgr.14397) 259 : cluster [DBG] pgmap v312: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 120 KiB/s rd, 6.9 MiB/s wr, 172 op/s 2024-09-13T00:36:29.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:29 smithi077 bash[22596]: cluster 2024-09-13T00:36:28.284329+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2024-09-13T00:36:29.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:29 smithi077 bash[22596]: audit 2024-09-13T00:36:28.855302+0000 mon.a (mon.0) 950 : audit [DBG] from='client.? 172.21.15.77:0/1878814252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:29.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:29 smithi077 bash[32040]: cluster 2024-09-13T00:36:27.902430+0000 mgr.y (mgr.14397) 259 : cluster [DBG] pgmap v312: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 120 KiB/s rd, 6.9 MiB/s wr, 172 op/s 2024-09-13T00:36:29.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:29 smithi077 bash[32040]: cluster 2024-09-13T00:36:28.284329+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2024-09-13T00:36:29.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:29 smithi077 bash[32040]: audit 2024-09-13T00:36:28.855302+0000 mon.a (mon.0) 950 : audit [DBG] from='client.? 172.21.15.77:0/1878814252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:29.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:29 smithi088 bash[24895]: cluster 2024-09-13T00:36:27.902430+0000 mgr.y (mgr.14397) 259 : cluster [DBG] pgmap v312: 137 pgs: 137 active+clean; 449 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 120 KiB/s rd, 6.9 MiB/s wr, 172 op/s 2024-09-13T00:36:29.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:29 smithi088 bash[24895]: cluster 2024-09-13T00:36:28.284329+0000 mon.a (mon.0) 949 : cluster [DBG] osdmap e151: 8 total, 8 up, 8 in 2024-09-13T00:36:29.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:29 smithi088 bash[24895]: audit 2024-09-13T00:36:28.855302+0000 mon.a (mon.0) 950 : audit [DBG] from='client.? 172.21.15.77:0/1878814252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:30.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:30 smithi088 bash[24895]: cluster 2024-09-13T00:36:29.293185+0000 mon.a (mon.0) 951 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2024-09-13T00:36:30.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:30 smithi077 bash[32040]: cluster 2024-09-13T00:36:29.293185+0000 mon.a (mon.0) 951 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2024-09-13T00:36:30.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:30 smithi077 bash[22596]: cluster 2024-09-13T00:36:29.293185+0000 mon.a (mon.0) 951 : cluster [DBG] osdmap e152: 8 total, 8 up, 8 in 2024-09-13T00:36:30.831 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.SnapsShrinkAfterPrepare (13122 ms) 2024-09-13T00:36:30.831 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroBeforePrepare 2024-09-13T00:36:30.912 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:36:30.941 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:36:31.031 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:36:31.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:30 smithi088 bash[25838]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:36:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:36:31.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:31 smithi088 bash[24895]: cluster 2024-09-13T00:36:29.903182+0000 mgr.y (mgr.14397) 260 : cluster [DBG] pgmap v315: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 5.2 MiB/s wr, 359 op/s 2024-09-13T00:36:31.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:31 smithi088 bash[24895]: cluster 2024-09-13T00:36:30.310294+0000 mon.a (mon.0) 952 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2024-09-13T00:36:31.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:31 smithi088 bash[24895]: audit 2024-09-13T00:36:31.138008+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:31.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:31 smithi088 bash[24895]: audit 2024-09-13T00:36:31.143346+0000 mon.c (mon.2) 147 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:36:31.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:31 smithi088 bash[24895]: audit 2024-09-13T00:36:31.143979+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:36:31.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:31 smithi088 bash[24895]: audit 2024-09-13T00:36:31.145932+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:36:31.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:31 smithi088 bash[24895]: audit 2024-09-13T00:36:31.147690+0000 mon.c (mon.2) 149 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:36:31.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:31 smithi088 bash[24895]: audit 2024-09-13T00:36:31.220974+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.77:0/4165089164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:31.775 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:36:31] "GET /metrics HTTP/1.1" 200 246372 "" "Prometheus/2.43.0" 2024-09-13T00:36:31.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[22596]: cluster 2024-09-13T00:36:29.903182+0000 mgr.y (mgr.14397) 260 : cluster [DBG] pgmap v315: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 5.2 MiB/s wr, 359 op/s 2024-09-13T00:36:31.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[22596]: cluster 2024-09-13T00:36:30.310294+0000 mon.a (mon.0) 952 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2024-09-13T00:36:31.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[22596]: audit 2024-09-13T00:36:31.138008+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:31.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[22596]: audit 2024-09-13T00:36:31.143346+0000 mon.c (mon.2) 147 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:36:31.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[22596]: audit 2024-09-13T00:36:31.143979+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:36:31.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[22596]: audit 2024-09-13T00:36:31.145932+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:36:31.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[22596]: audit 2024-09-13T00:36:31.147690+0000 mon.c (mon.2) 149 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:36:31.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[22596]: audit 2024-09-13T00:36:31.220974+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.77:0/4165089164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:31.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[32040]: cluster 2024-09-13T00:36:29.903182+0000 mgr.y (mgr.14397) 260 : cluster [DBG] pgmap v315: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 5.2 MiB/s wr, 359 op/s 2024-09-13T00:36:31.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[32040]: cluster 2024-09-13T00:36:30.310294+0000 mon.a (mon.0) 952 : cluster [DBG] osdmap e153: 8 total, 8 up, 8 in 2024-09-13T00:36:31.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[32040]: audit 2024-09-13T00:36:31.138008+0000 mon.a (mon.0) 953 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:31.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[32040]: audit 2024-09-13T00:36:31.143346+0000 mon.c (mon.2) 147 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:36:31.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[32040]: audit 2024-09-13T00:36:31.143979+0000 mon.a (mon.0) 954 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:36:31.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[32040]: audit 2024-09-13T00:36:31.145932+0000 mon.c (mon.2) 148 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:36:31.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[32040]: audit 2024-09-13T00:36:31.147690+0000 mon.c (mon.2) 149 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:36:31.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:31 smithi077 bash[32040]: audit 2024-09-13T00:36:31.220974+0000 mon.c (mon.2) 150 : audit [DBG] from='client.? 172.21.15.77:0/4165089164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:32.344 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 8388609~8388609 2024-09-13T00:36:32.376 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 0~1000 2024-09-13T00:36:32.376 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 1010~1000 2024-09-13T00:36:32.376 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap2 2024-09-13T00:36:32.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:32 smithi088 bash[24895]: cephadm 2024-09-13T00:36:31.127588+0000 mgr.y (mgr.14397) 261 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-13T00:36:32.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:32 smithi088 bash[24895]: cephadm 2024-09-13T00:36:31.149404+0000 mgr.y (mgr.14397) 262 : cephadm [INF] Deploying daemon mgr.x on smithi088 2024-09-13T00:36:32.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:32 smithi088 bash[24895]: cluster 2024-09-13T00:36:31.319418+0000 mon.a (mon.0) 955 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2024-09-13T00:36:32.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:32 smithi077 bash[22596]: cephadm 2024-09-13T00:36:31.127588+0000 mgr.y (mgr.14397) 261 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-13T00:36:32.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:32 smithi077 bash[22596]: cephadm 2024-09-13T00:36:31.149404+0000 mgr.y (mgr.14397) 262 : cephadm [INF] Deploying daemon mgr.x on smithi088 2024-09-13T00:36:32.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:32 smithi077 bash[22596]: cluster 2024-09-13T00:36:31.319418+0000 mon.a (mon.0) 955 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2024-09-13T00:36:32.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:32 smithi077 bash[32040]: cephadm 2024-09-13T00:36:31.127588+0000 mgr.y (mgr.14397) 261 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-13T00:36:32.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:32 smithi077 bash[32040]: cephadm 2024-09-13T00:36:31.149404+0000 mgr.y (mgr.14397) 262 : cephadm [INF] Deploying daemon mgr.x on smithi088 2024-09-13T00:36:32.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:32 smithi077 bash[32040]: cluster 2024-09-13T00:36:31.319418+0000 mon.a (mon.0) 955 : cluster [DBG] osdmap e154: 8 total, 8 up, 8 in 2024-09-13T00:36:33.271 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:36:33 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:33.393 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:33 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:33.393 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:36:33 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:33.394 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:36:33 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:33.394 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:36:33 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:33.395 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 00:36:33 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:33.395 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:36:33 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:33.395 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:36:33 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:33.395 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:33 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:33.396 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:33 smithi088 bash[24895]: cluster 2024-09-13T00:36:31.903882+0000 mgr.y (mgr.14397) 263 : cluster [DBG] pgmap v318: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 418 op/s 2024-09-13T00:36:33.396 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:33 smithi088 bash[24895]: cluster 2024-09-13T00:36:32.331838+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2024-09-13T00:36:33.738 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:33 smithi088 systemd[1]: Stopping Ceph mgr.x for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:36:33.776 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:36:33 smithi077 bash[58266]: ts=2024-09-13T00:36:33.533Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": read tcp 172.21.15.77:40584->172.21.15.88:8443: read: connection reset by peer" 2024-09-13T00:36:33.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:33 smithi077 bash[22596]: cluster 2024-09-13T00:36:31.903882+0000 mgr.y (mgr.14397) 263 : cluster [DBG] pgmap v318: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 418 op/s 2024-09-13T00:36:33.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:33 smithi077 bash[22596]: cluster 2024-09-13T00:36:32.331838+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2024-09-13T00:36:33.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:33 smithi077 bash[32040]: cluster 2024-09-13T00:36:31.903882+0000 mgr.y (mgr.14397) 263 : cluster [DBG] pgmap v318: 137 pgs: 137 active+clean; 469 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 418 op/s 2024-09-13T00:36:33.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:33 smithi077 bash[32040]: cluster 2024-09-13T00:36:32.331838+0000 mon.a (mon.0) 956 : cluster [DBG] osdmap e155: 8 total, 8 up, 8 in 2024-09-13T00:36:34.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:33 smithi088 bash[53463]: ceph-9218306c-7165-11ef-bceb-c7b262605968-mgr-x 2024-09-13T00:36:34.369 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 8388609~1000 2024-09-13T00:36:34.385 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 16777228~1000 2024-09-13T00:36:34.386 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 167772160 2024-09-13T00:36:34.404 INFO:tasks.workunit.client.0.smithi077.stdout:write: 2 83886079~8388609 2024-09-13T00:36:34.449 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:36:34 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:34.450 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:34 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:34.450 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:34 smithi088 bash[24895]: cluster 2024-09-13T00:36:33.331758+0000 mon.a (mon.0) 957 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2024-09-13T00:36:34.450 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:34 smithi088 bash[24895]: audit 2024-09-13T00:36:33.581886+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.77:0/3440222499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:34.450 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:36:34 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:34.451 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:36:34 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:34.451 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:36:34 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:34.451 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 00:36:34 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:34.452 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:36:34 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:34.452 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:36:34 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:34.452 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:34 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T00:36:34.452 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:34 smithi088 bash[53524]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-mgr-x 2024-09-13T00:36:34.452 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:34 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T00:36:34.453 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:34 smithi088 systemd[1]: Stopped Ceph mgr.x for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:36:34.453 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:34 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:36:34.453 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:34 smithi088 systemd[1]: Started Ceph mgr.x for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:36:34.453 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap3 2024-09-13T00:36:34.528 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:36:34 smithi077 bash[58266]: ts=2024-09-13T00:36:34.089Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=2 err="Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused" 2024-09-13T00:36:34.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:34 smithi077 bash[22596]: cluster 2024-09-13T00:36:33.331758+0000 mon.a (mon.0) 957 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2024-09-13T00:36:34.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:34 smithi077 bash[22596]: audit 2024-09-13T00:36:33.581886+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.77:0/3440222499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:34 smithi077 bash[32040]: cluster 2024-09-13T00:36:33.331758+0000 mon.a (mon.0) 957 : cluster [DBG] osdmap e156: 8 total, 8 up, 8 in 2024-09-13T00:36:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:34 smithi077 bash[32040]: audit 2024-09-13T00:36:33.581886+0000 mon.a (mon.0) 958 : audit [DBG] from='client.? 172.21.15.77:0/3440222499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:35.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:35 smithi088 bash[24895]: cluster 2024-09-13T00:36:33.905423+0000 mgr.y (mgr.14397) 264 : cluster [DBG] pgmap v321: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 145 op/s 2024-09-13T00:36:35.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:35 smithi088 bash[24895]: cluster 2024-09-13T00:36:34.352730+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2024-09-13T00:36:35.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:35 smithi088 bash[24895]: audit 2024-09-13T00:36:34.511513+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:35.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:35 smithi088 bash[24895]: audit 2024-09-13T00:36:34.523614+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:35.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:35 smithi088 bash[24895]: audit 2024-09-13T00:36:34.526816+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:35.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:35 smithi077 bash[32040]: cluster 2024-09-13T00:36:33.905423+0000 mgr.y (mgr.14397) 264 : cluster [DBG] pgmap v321: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 145 op/s 2024-09-13T00:36:35.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:35 smithi077 bash[32040]: cluster 2024-09-13T00:36:34.352730+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2024-09-13T00:36:35.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:35 smithi077 bash[32040]: audit 2024-09-13T00:36:34.511513+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:35.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:35 smithi077 bash[32040]: audit 2024-09-13T00:36:34.523614+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:35.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:35 smithi077 bash[32040]: audit 2024-09-13T00:36:34.526816+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:35.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:35 smithi077 bash[22596]: cluster 2024-09-13T00:36:33.905423+0000 mgr.y (mgr.14397) 264 : cluster [DBG] pgmap v321: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 145 op/s 2024-09-13T00:36:35.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:35 smithi077 bash[22596]: cluster 2024-09-13T00:36:34.352730+0000 mon.a (mon.0) 959 : cluster [DBG] osdmap e157: 8 total, 8 up, 8 in 2024-09-13T00:36:35.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:35 smithi077 bash[22596]: audit 2024-09-13T00:36:34.511513+0000 mon.a (mon.0) 960 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:35.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:35 smithi077 bash[22596]: audit 2024-09-13T00:36:34.523614+0000 mon.a (mon.0) 961 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:35.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:35 smithi077 bash[22596]: audit 2024-09-13T00:36:34.526816+0000 mon.c (mon.2) 151 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:36.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:35 smithi088 bash[53580]: debug 2024-09-13T00:36:35.604+0000 7f0393031640 1 -- 172.21.15.88:0/3783993298 <== mon.1 v2:172.21.15.88:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b53cdb8d00 con 0x55b53cdaa800 2024-09-13T00:36:36.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:35 smithi088 bash[53580]: debug 2024-09-13T00:36:35.716+0000 7f039589a040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T00:36:36.156 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:36:36.201 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 83886079~1 2024-09-13T00:36:36.220 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 0 2024-09-13T00:36:36.256 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:36:36.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:36 smithi077 bash[22596]: cluster 2024-09-13T00:36:35.518661+0000 mon.a (mon.0) 962 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2024-09-13T00:36:36.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:36 smithi077 bash[22596]: audit 2024-09-13T00:36:35.927781+0000 mon.a (mon.0) 963 : audit [DBG] from='client.? 172.21.15.77:0/696295685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:36.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:36 smithi077 bash[22596]: cluster 2024-09-13T00:36:36.131194+0000 mon.a (mon.0) 964 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2024-09-13T00:36:36.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:36 smithi077 bash[32040]: cluster 2024-09-13T00:36:35.518661+0000 mon.a (mon.0) 962 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2024-09-13T00:36:36.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:36 smithi077 bash[32040]: audit 2024-09-13T00:36:35.927781+0000 mon.a (mon.0) 963 : audit [DBG] from='client.? 172.21.15.77:0/696295685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:36.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:36 smithi077 bash[32040]: cluster 2024-09-13T00:36:36.131194+0000 mon.a (mon.0) 964 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2024-09-13T00:36:36.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:36 smithi088 bash[24895]: cluster 2024-09-13T00:36:35.518661+0000 mon.a (mon.0) 962 : cluster [DBG] osdmap e158: 8 total, 8 up, 8 in 2024-09-13T00:36:36.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:36 smithi088 bash[24895]: audit 2024-09-13T00:36:35.927781+0000 mon.a (mon.0) 963 : audit [DBG] from='client.? 172.21.15.77:0/696295685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:36.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:36 smithi088 bash[24895]: cluster 2024-09-13T00:36:36.131194+0000 mon.a (mon.0) 964 : cluster [DBG] osdmap e159: 8 total, 8 up, 8 in 2024-09-13T00:36:37.350 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:36 smithi088 bash[53580]: debug 2024-09-13T00:36:36.948+0000 7f039589a040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T00:36:37.350 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:37 smithi088 bash[53580]: debug 2024-09-13T00:36:37.012+0000 7f039589a040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T00:36:37.350 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:37 smithi088 bash[53580]: debug 2024-09-13T00:36:37.164+0000 7f039589a040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T00:36:37.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:37 smithi077 bash[22596]: cluster 2024-09-13T00:36:35.906116+0000 mgr.y (mgr.14397) 265 : cluster [DBG] pgmap v324: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 145 op/s 2024-09-13T00:36:37.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:37 smithi077 bash[22596]: cluster 2024-09-13T00:36:37.137378+0000 mon.a (mon.0) 965 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2024-09-13T00:36:37.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:37 smithi077 bash[32040]: cluster 2024-09-13T00:36:35.906116+0000 mgr.y (mgr.14397) 265 : cluster [DBG] pgmap v324: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 145 op/s 2024-09-13T00:36:37.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:37 smithi077 bash[32040]: cluster 2024-09-13T00:36:37.137378+0000 mon.a (mon.0) 965 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2024-09-13T00:36:37.804 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:37 smithi088 bash[24895]: cluster 2024-09-13T00:36:35.906116+0000 mgr.y (mgr.14397) 265 : cluster [DBG] pgmap v324: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 145 op/s 2024-09-13T00:36:37.804 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:37 smithi088 bash[24895]: cluster 2024-09-13T00:36:37.137378+0000 mon.a (mon.0) 965 : cluster [DBG] osdmap e160: 8 total, 8 up, 8 in 2024-09-13T00:36:38.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:37 smithi088 bash[53580]: debug 2024-09-13T00:36:37.804+0000 7f039589a040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T00:36:38.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:37 smithi088 bash[53580]: debug 2024-09-13T00:36:37.876+0000 7f039589a040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T00:36:38.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:37 smithi088 bash[53580]: debug 2024-09-13T00:36:37.944+0000 7f039589a040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T00:36:38.523 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:38 smithi088 bash[53580]: debug 2024-09-13T00:36:38.124+0000 7f039589a040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T00:36:38.523 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:38 smithi088 bash[53580]: debug 2024-09-13T00:36:38.204+0000 7f039589a040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T00:36:38.523 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:38 smithi088 bash[53580]: debug 2024-09-13T00:36:38.272+0000 7f039589a040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T00:36:38.780 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:38 smithi088 bash[53580]: debug 2024-09-13T00:36:38.524+0000 7f039589a040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T00:36:38.780 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:38 smithi088 bash[53580]: debug 2024-09-13T00:36:38.592+0000 7f039589a040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T00:36:39.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:38 smithi088 bash[53580]: debug 2024-09-13T00:36:38.780+0000 7f039589a040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T00:36:39.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:38 smithi088 bash[53580]: debug 2024-09-13T00:36:38.860+0000 7f039589a040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T00:36:39.490 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:39 smithi088 bash[24895]: cluster 2024-09-13T00:36:37.906817+0000 mgr.y (mgr.14397) 266 : cluster [DBG] pgmap v327: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:36:39.490 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:39 smithi088 bash[24895]: cluster 2024-09-13T00:36:38.136433+0000 mon.a (mon.0) 966 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2024-09-13T00:36:39.490 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:39 smithi088 bash[24895]: audit 2024-09-13T00:36:38.283946+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.77:0/4065688639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:39.490 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:39 smithi088 bash[53580]: debug 2024-09-13T00:36:39.116+0000 7f039589a040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T00:36:39.490 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:39 smithi088 bash[53580]: debug 2024-09-13T00:36:39.312+0000 7f039589a040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:36:39.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:39 smithi077 bash[32040]: cluster 2024-09-13T00:36:37.906817+0000 mgr.y (mgr.14397) 266 : cluster [DBG] pgmap v327: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:36:39.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:39 smithi077 bash[32040]: cluster 2024-09-13T00:36:38.136433+0000 mon.a (mon.0) 966 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2024-09-13T00:36:39.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:39 smithi077 bash[32040]: audit 2024-09-13T00:36:38.283946+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.77:0/4065688639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:39.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:39 smithi077 bash[22596]: cluster 2024-09-13T00:36:37.906817+0000 mgr.y (mgr.14397) 266 : cluster [DBG] pgmap v327: 137 pgs: 137 active+clean; 481 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:36:39.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:39 smithi077 bash[22596]: cluster 2024-09-13T00:36:38.136433+0000 mon.a (mon.0) 966 : cluster [DBG] osdmap e161: 8 total, 8 up, 8 in 2024-09-13T00:36:39.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:39 smithi077 bash[22596]: audit 2024-09-13T00:36:38.283946+0000 mon.c (mon.2) 152 : audit [DBG] from='client.? 172.21.15.77:0/4065688639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:39.689 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:36:39.806 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:39 smithi088 bash[53580]: debug 2024-09-13T00:36:39.492+0000 7f039589a040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T00:36:39.806 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:39 smithi088 bash[53580]: debug 2024-09-13T00:36:39.580+0000 7f039589a040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T00:36:39.880 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:36:40.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:39 smithi088 bash[53580]: debug 2024-09-13T00:36:39.808+0000 7f039589a040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:36:40.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:40 smithi077 bash[32040]: cluster 2024-09-13T00:36:39.149520+0000 mon.a (mon.0) 967 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2024-09-13T00:36:40.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:40 smithi077 bash[32040]: audit 2024-09-13T00:36:39.938193+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:40.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:40 smithi077 bash[32040]: audit 2024-09-13T00:36:39.948106+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:40.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:40 smithi077 bash[22596]: cluster 2024-09-13T00:36:39.149520+0000 mon.a (mon.0) 967 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2024-09-13T00:36:40.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:40 smithi077 bash[22596]: audit 2024-09-13T00:36:39.938193+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:40.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:40 smithi077 bash[22596]: audit 2024-09-13T00:36:39.948106+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:40.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:40 smithi088 bash[24895]: cluster 2024-09-13T00:36:39.149520+0000 mon.a (mon.0) 967 : cluster [DBG] osdmap e162: 8 total, 8 up, 8 in 2024-09-13T00:36:40.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:40 smithi088 bash[24895]: audit 2024-09-13T00:36:39.938193+0000 mon.a (mon.0) 968 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:40.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:40 smithi088 bash[24895]: audit 2024-09-13T00:36:39.948106+0000 mon.a (mon.0) 969 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:41.011 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:40 smithi088 bash[53580]: debug 2024-09-13T00:36:40.696+0000 7f039589a040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T00:36:41.011 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:40 smithi088 bash[53580]: debug 2024-09-13T00:36:40.772+0000 7f039589a040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T00:36:41.350 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:41 smithi088 bash[53580]: debug 2024-09-13T00:36:41.012+0000 7f039589a040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T00:36:41.350 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:41 smithi088 bash[53580]: debug 2024-09-13T00:36:41.088+0000 7f039589a040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T00:36:41.667 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:41 smithi088 bash[24895]: cluster 2024-09-13T00:36:39.907272+0000 mgr.y (mgr.14397) 267 : cluster [DBG] pgmap v330: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 126 KiB/s rd, 4.0 MiB/s wr, 184 op/s 2024-09-13T00:36:41.667 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:41 smithi088 bash[24895]: audit 2024-09-13T00:36:40.372549+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:41.667 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:41 smithi088 bash[24895]: audit 2024-09-13T00:36:40.385386+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:41.667 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:41 smithi088 bash[24895]: audit 2024-09-13T00:36:40.629224+0000 mon.a (mon.0) 972 : audit [DBG] from='client.? 172.21.15.77:0/2774540264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:41.667 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:41 smithi088 bash[24895]: cluster 2024-09-13T00:36:40.945343+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2024-09-13T00:36:41.667 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:41 smithi088 bash[24895]: audit 2024-09-13T00:36:40.993986+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:41.668 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:41 smithi088 bash[24895]: audit 2024-09-13T00:36:41.005981+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:41.668 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:41 smithi088 bash[53580]: debug 2024-09-13T00:36:41.480+0000 7f039589a040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T00:36:41.668 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:41 smithi088 bash[53580]: debug 2024-09-13T00:36:41.560+0000 7f039589a040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T00:36:41.668 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:41 smithi088 bash[53580]: [13/Sep/2024:00:36:41] ENGINE Bus STARTING 2024-09-13T00:36:41.668 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:41 smithi088 bash[53580]: CherryPy Checker: 2024-09-13T00:36:41.668 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:41 smithi088 bash[53580]: The Application mounted at '' has an empty config. 2024-09-13T00:36:41.668 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:41 smithi088 bash[53580]: [13/Sep/2024:00:36:41] ENGINE Serving on http://:::9283 2024-09-13T00:36:41.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:41 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:36:41] "GET /metrics HTTP/1.1" 200 246372 "" "Prometheus/2.43.0" 2024-09-13T00:36:41.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:41 smithi077 bash[22596]: cluster 2024-09-13T00:36:39.907272+0000 mgr.y (mgr.14397) 267 : cluster [DBG] pgmap v330: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 126 KiB/s rd, 4.0 MiB/s wr, 184 op/s 2024-09-13T00:36:41.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:41 smithi077 bash[22596]: audit 2024-09-13T00:36:40.372549+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:41.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:41 smithi077 bash[22596]: audit 2024-09-13T00:36:40.385386+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:41.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:41 smithi077 bash[22596]: audit 2024-09-13T00:36:40.629224+0000 mon.a (mon.0) 972 : audit [DBG] from='client.? 172.21.15.77:0/2774540264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:41.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:41 smithi077 bash[22596]: cluster 2024-09-13T00:36:40.945343+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2024-09-13T00:36:41.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:41 smithi077 bash[22596]: audit 2024-09-13T00:36:40.993986+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:41.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:41 smithi077 bash[22596]: audit 2024-09-13T00:36:41.005981+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:41.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:41 smithi077 bash[32040]: cluster 2024-09-13T00:36:39.907272+0000 mgr.y (mgr.14397) 267 : cluster [DBG] pgmap v330: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 126 KiB/s rd, 4.0 MiB/s wr, 184 op/s 2024-09-13T00:36:41.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:41 smithi077 bash[32040]: audit 2024-09-13T00:36:40.372549+0000 mon.a (mon.0) 970 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:41.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:41 smithi077 bash[32040]: audit 2024-09-13T00:36:40.385386+0000 mon.a (mon.0) 971 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:41.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:41 smithi077 bash[32040]: audit 2024-09-13T00:36:40.629224+0000 mon.a (mon.0) 972 : audit [DBG] from='client.? 172.21.15.77:0/2774540264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:41.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:41 smithi077 bash[32040]: cluster 2024-09-13T00:36:40.945343+0000 mon.a (mon.0) 973 : cluster [DBG] osdmap e163: 8 total, 8 up, 8 in 2024-09-13T00:36:41.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:41 smithi077 bash[32040]: audit 2024-09-13T00:36:40.993986+0000 mon.a (mon.0) 974 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:41.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:41 smithi077 bash[32040]: audit 2024-09-13T00:36:41.005981+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:42.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:41 smithi088 bash[53580]: [13/Sep/2024:00:36:41] ENGINE Bus STARTED 2024-09-13T00:36:42.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:42 smithi077 bash[32040]: cluster 2024-09-13T00:36:41.568767+0000 mon.a (mon.0) 976 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T00:36:42.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:42 smithi077 bash[32040]: cluster 2024-09-13T00:36:41.569011+0000 mon.a (mon.0) 977 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:36:42.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:42 smithi077 bash[32040]: audit 2024-09-13T00:36:41.569409+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.? 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:36:42.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:42 smithi077 bash[32040]: audit 2024-09-13T00:36:41.570652+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:36:42.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:42 smithi077 bash[32040]: audit 2024-09-13T00:36:41.572916+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.? 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:36:42.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:42 smithi077 bash[32040]: audit 2024-09-13T00:36:41.573264+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:36:42.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:42 smithi077 bash[32040]: cluster 2024-09-13T00:36:41.948035+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2024-09-13T00:36:42.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:42 smithi077 bash[22596]: cluster 2024-09-13T00:36:41.568767+0000 mon.a (mon.0) 976 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T00:36:42.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:42 smithi077 bash[22596]: cluster 2024-09-13T00:36:41.569011+0000 mon.a (mon.0) 977 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:36:42.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:42 smithi077 bash[22596]: audit 2024-09-13T00:36:41.569409+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.? 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:36:42.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:42 smithi077 bash[22596]: audit 2024-09-13T00:36:41.570652+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:36:42.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:42 smithi077 bash[22596]: audit 2024-09-13T00:36:41.572916+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.? 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:36:42.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:42 smithi077 bash[22596]: audit 2024-09-13T00:36:41.573264+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:36:42.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:42 smithi077 bash[22596]: cluster 2024-09-13T00:36:41.948035+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2024-09-13T00:36:42.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:42 smithi088 bash[24895]: cluster 2024-09-13T00:36:41.568767+0000 mon.a (mon.0) 976 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T00:36:42.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:42 smithi088 bash[24895]: cluster 2024-09-13T00:36:41.569011+0000 mon.a (mon.0) 977 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:36:42.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:42 smithi088 bash[24895]: audit 2024-09-13T00:36:41.569409+0000 mon.b (mon.1) 30 : audit [DBG] from='mgr.? 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:36:42.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:42 smithi088 bash[24895]: audit 2024-09-13T00:36:41.570652+0000 mon.b (mon.1) 31 : audit [DBG] from='mgr.? 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:36:42.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:42 smithi088 bash[24895]: audit 2024-09-13T00:36:41.572916+0000 mon.b (mon.1) 32 : audit [DBG] from='mgr.? 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:36:42.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:42 smithi088 bash[24895]: audit 2024-09-13T00:36:41.573264+0000 mon.b (mon.1) 33 : audit [DBG] from='mgr.? 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:36:42.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:42 smithi088 bash[24895]: cluster 2024-09-13T00:36:41.948035+0000 mon.a (mon.0) 978 : cluster [DBG] osdmap e164: 8 total, 8 up, 8 in 2024-09-13T00:36:43.275 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:36:42 smithi077 bash[58266]: ts=2024-09-13T00:36:42.882Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=7 2024-09-13T00:36:43.374 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.SnapsShrinkToZeroBeforePrepare (12543 ms) 2024-09-13T00:36:43.374 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroAfterPrepare 2024-09-13T00:36:43.456 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:36:43.484 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:36:43.591 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:36:43.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:43 smithi077 bash[22596]: cluster 2024-09-13T00:36:41.907945+0000 mgr.y (mgr.14397) 268 : cluster [DBG] pgmap v332: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 106 KiB/s rd, 3.4 MiB/s wr, 154 op/s 2024-09-13T00:36:43.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:43 smithi077 bash[22596]: cluster 2024-09-13T00:36:42.401047+0000 mon.a (mon.0) 979 : cluster [DBG] mgrmap e21: y(active, since 7m), standbys: x 2024-09-13T00:36:43.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:43 smithi077 bash[22596]: audit 2024-09-13T00:36:42.838304+0000 mon.c (mon.2) 153 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:43.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:43 smithi077 bash[22596]: cluster 2024-09-13T00:36:42.952604+0000 mon.a (mon.0) 980 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2024-09-13T00:36:43.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:43 smithi077 bash[22596]: audit 2024-09-13T00:36:42.993734+0000 mon.a (mon.0) 981 : audit [DBG] from='client.? 172.21.15.77:0/3412918369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:43.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:43 smithi077 bash[32040]: cluster 2024-09-13T00:36:41.907945+0000 mgr.y (mgr.14397) 268 : cluster [DBG] pgmap v332: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 106 KiB/s rd, 3.4 MiB/s wr, 154 op/s 2024-09-13T00:36:43.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:43 smithi077 bash[32040]: cluster 2024-09-13T00:36:42.401047+0000 mon.a (mon.0) 979 : cluster [DBG] mgrmap e21: y(active, since 7m), standbys: x 2024-09-13T00:36:43.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:43 smithi077 bash[32040]: audit 2024-09-13T00:36:42.838304+0000 mon.c (mon.2) 153 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:43.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:43 smithi077 bash[32040]: cluster 2024-09-13T00:36:42.952604+0000 mon.a (mon.0) 980 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2024-09-13T00:36:43.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:43 smithi077 bash[32040]: audit 2024-09-13T00:36:42.993734+0000 mon.a (mon.0) 981 : audit [DBG] from='client.? 172.21.15.77:0/3412918369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:43.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:43 smithi088 bash[24895]: cluster 2024-09-13T00:36:41.907945+0000 mgr.y (mgr.14397) 268 : cluster [DBG] pgmap v332: 137 pgs: 137 active+clean; 505 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 106 KiB/s rd, 3.4 MiB/s wr, 154 op/s 2024-09-13T00:36:43.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:43 smithi088 bash[24895]: cluster 2024-09-13T00:36:42.401047+0000 mon.a (mon.0) 979 : cluster [DBG] mgrmap e21: y(active, since 7m), standbys: x 2024-09-13T00:36:43.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:43 smithi088 bash[24895]: audit 2024-09-13T00:36:42.838304+0000 mon.c (mon.2) 153 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:43.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:43 smithi088 bash[24895]: cluster 2024-09-13T00:36:42.952604+0000 mon.a (mon.0) 980 : cluster [DBG] osdmap e165: 8 total, 8 up, 8 in 2024-09-13T00:36:43.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:43 smithi088 bash[24895]: audit 2024-09-13T00:36:42.993734+0000 mon.a (mon.0) 981 : audit [DBG] from='client.? 172.21.15.77:0/3412918369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:45.419 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 8388609~8388609 2024-09-13T00:36:45.451 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 0~1000 2024-09-13T00:36:45.451 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 1010~1000 2024-09-13T00:36:45.451 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap2 2024-09-13T00:36:45.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:45 smithi077 bash[22596]: cluster 2024-09-13T00:36:43.909492+0000 mgr.y (mgr.14397) 269 : cluster [DBG] pgmap v335: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 8.4 MiB/s wr, 397 op/s 2024-09-13T00:36:45.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:45 smithi077 bash[22596]: cluster 2024-09-13T00:36:44.406062+0000 mon.a (mon.0) 982 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2024-09-13T00:36:45.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:45 smithi077 bash[22596]: audit 2024-09-13T00:36:45.341203+0000 mon.c (mon.2) 154 : audit [DBG] from='client.? 172.21.15.77:0/4232368316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:45.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:45 smithi077 bash[32040]: cluster 2024-09-13T00:36:43.909492+0000 mgr.y (mgr.14397) 269 : cluster [DBG] pgmap v335: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 8.4 MiB/s wr, 397 op/s 2024-09-13T00:36:45.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:45 smithi077 bash[32040]: cluster 2024-09-13T00:36:44.406062+0000 mon.a (mon.0) 982 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2024-09-13T00:36:45.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:45 smithi077 bash[32040]: audit 2024-09-13T00:36:45.341203+0000 mon.c (mon.2) 154 : audit [DBG] from='client.? 172.21.15.77:0/4232368316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:45.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:45 smithi088 bash[24895]: cluster 2024-09-13T00:36:43.909492+0000 mgr.y (mgr.14397) 269 : cluster [DBG] pgmap v335: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 46 MiB/s rd, 8.4 MiB/s wr, 397 op/s 2024-09-13T00:36:45.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:45 smithi088 bash[24895]: cluster 2024-09-13T00:36:44.406062+0000 mon.a (mon.0) 982 : cluster [DBG] osdmap e166: 8 total, 8 up, 8 in 2024-09-13T00:36:45.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:45 smithi088 bash[24895]: audit 2024-09-13T00:36:45.341203+0000 mon.c (mon.2) 154 : audit [DBG] from='client.? 172.21.15.77:0/4232368316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:46.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:46 smithi077 bash[22596]: cluster 2024-09-13T00:36:45.405000+0000 mon.a (mon.0) 983 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2024-09-13T00:36:46.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:46 smithi077 bash[22596]: cluster 2024-09-13T00:36:46.137365+0000 mon.a (mon.0) 984 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2024-09-13T00:36:46.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:46 smithi077 bash[32040]: cluster 2024-09-13T00:36:45.405000+0000 mon.a (mon.0) 983 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2024-09-13T00:36:46.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:46 smithi077 bash[32040]: cluster 2024-09-13T00:36:46.137365+0000 mon.a (mon.0) 984 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2024-09-13T00:36:46.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:46 smithi088 bash[24895]: cluster 2024-09-13T00:36:45.405000+0000 mon.a (mon.0) 983 : cluster [DBG] osdmap e167: 8 total, 8 up, 8 in 2024-09-13T00:36:46.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:46 smithi088 bash[24895]: cluster 2024-09-13T00:36:46.137365+0000 mon.a (mon.0) 984 : cluster [DBG] osdmap e168: 8 total, 8 up, 8 in 2024-09-13T00:36:47.154 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 8388609~1000 2024-09-13T00:36:47.165 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 16777228~1000 2024-09-13T00:36:47.165 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 167772160 2024-09-13T00:36:47.183 INFO:tasks.workunit.client.0.smithi077.stdout:write: 2 83886079~8388609 2024-09-13T00:36:47.234 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap3 2024-09-13T00:36:47.775 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:47 smithi077 bash[22596]: cluster 2024-09-13T00:36:45.910224+0000 mgr.y (mgr.14397) 270 : cluster [DBG] pgmap v338: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 54 MiB/s rd, 10 MiB/s wr, 473 op/s 2024-09-13T00:36:47.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:47 smithi077 bash[22596]: cluster 2024-09-13T00:36:47.138283+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2024-09-13T00:36:47.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:47 smithi077 bash[32040]: cluster 2024-09-13T00:36:45.910224+0000 mgr.y (mgr.14397) 270 : cluster [DBG] pgmap v338: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 54 MiB/s rd, 10 MiB/s wr, 473 op/s 2024-09-13T00:36:47.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:47 smithi077 bash[32040]: cluster 2024-09-13T00:36:47.138283+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2024-09-13T00:36:47.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:47 smithi088 bash[24895]: cluster 2024-09-13T00:36:45.910224+0000 mgr.y (mgr.14397) 270 : cluster [DBG] pgmap v338: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 54 MiB/s rd, 10 MiB/s wr, 473 op/s 2024-09-13T00:36:47.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:47 smithi088 bash[24895]: cluster 2024-09-13T00:36:47.138283+0000 mon.a (mon.0) 985 : cluster [DBG] osdmap e169: 8 total, 8 up, 8 in 2024-09-13T00:36:48.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:48 smithi077 bash[22596]: audit 2024-09-13T00:36:47.692551+0000 mon.a (mon.0) 986 : audit [DBG] from='client.? 172.21.15.77:0/1847990641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:48.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:48 smithi077 bash[22596]: cluster 2024-09-13T00:36:48.141477+0000 mon.a (mon.0) 987 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2024-09-13T00:36:48.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:48 smithi077 bash[32040]: audit 2024-09-13T00:36:47.692551+0000 mon.a (mon.0) 986 : audit [DBG] from='client.? 172.21.15.77:0/1847990641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:48.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:48 smithi077 bash[32040]: cluster 2024-09-13T00:36:48.141477+0000 mon.a (mon.0) 987 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2024-09-13T00:36:48.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:48 smithi088 bash[24895]: audit 2024-09-13T00:36:47.692551+0000 mon.a (mon.0) 986 : audit [DBG] from='client.? 172.21.15.77:0/1847990641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:48.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:48 smithi088 bash[24895]: cluster 2024-09-13T00:36:48.141477+0000 mon.a (mon.0) 987 : cluster [DBG] osdmap e170: 8 total, 8 up, 8 in 2024-09-13T00:36:49.159 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:36:49.207 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 83886079~1 2024-09-13T00:36:49.226 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:36:49.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:49 smithi077 bash[22596]: cluster 2024-09-13T00:36:47.911002+0000 mgr.y (mgr.14397) 271 : cluster [DBG] pgmap v341: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:36:49.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:49 smithi077 bash[22596]: cluster 2024-09-13T00:36:49.143943+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2024-09-13T00:36:49.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:49 smithi077 bash[32040]: cluster 2024-09-13T00:36:47.911002+0000 mgr.y (mgr.14397) 271 : cluster [DBG] pgmap v341: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:36:49.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:49 smithi077 bash[32040]: cluster 2024-09-13T00:36:49.143943+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2024-09-13T00:36:49.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:49 smithi088 bash[24895]: cluster 2024-09-13T00:36:47.911002+0000 mgr.y (mgr.14397) 271 : cluster [DBG] pgmap v341: 137 pgs: 137 active+clean; 521 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:36:49.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:49 smithi088 bash[24895]: cluster 2024-09-13T00:36:49.143943+0000 mon.a (mon.0) 988 : cluster [DBG] osdmap e171: 8 total, 8 up, 8 in 2024-09-13T00:36:50.760 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:50 smithi088 bash[24895]: audit 2024-09-13T00:36:50.038718+0000 mon.c (mon.2) 155 : audit [DBG] from='client.? 172.21.15.77:0/3840567612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:50.761 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:50 smithi088 bash[24895]: cluster 2024-09-13T00:36:50.145189+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2024-09-13T00:36:50.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:50 smithi077 bash[32040]: audit 2024-09-13T00:36:50.038718+0000 mon.c (mon.2) 155 : audit [DBG] from='client.? 172.21.15.77:0/3840567612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:50.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:50 smithi077 bash[32040]: cluster 2024-09-13T00:36:50.145189+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2024-09-13T00:36:50.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:50 smithi077 bash[22596]: audit 2024-09-13T00:36:50.038718+0000 mon.c (mon.2) 155 : audit [DBG] from='client.? 172.21.15.77:0/3840567612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:50.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:50 smithi077 bash[22596]: cluster 2024-09-13T00:36:50.145189+0000 mon.a (mon.0) 989 : cluster [DBG] osdmap e172: 8 total, 8 up, 8 in 2024-09-13T00:36:51.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:50 smithi088 bash[53580]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:36:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:36:51.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:50 smithi088 bash[53580]: [13/Sep/2024:00:36:50] ENGINE Bus STOPPING 2024-09-13T00:36:51.276 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:50 smithi077 bash[22845]: ignoring --setuser ceph since I am not root 2024-09-13T00:36:51.276 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:50 smithi077 bash[22845]: ignoring --setgroup ceph since I am not root 2024-09-13T00:36:51.276 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:50 smithi077 bash[22845]: debug 2024-09-13T00:36:50.837+0000 7f98f6159640 1 -- 172.21.15.77:0/3225327678 <== mon.2 v2:172.21.15.77:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560bf5bd8340 con 0x560bf5be4c00 2024-09-13T00:36:51.276 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:50 smithi077 bash[22845]: debug 2024-09-13T00:36:50.941+0000 7f98f89b3ec0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T00:36:51.536 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[53580]: [13/Sep/2024:00:36:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:36:51.536 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[53580]: [13/Sep/2024:00:36:51] ENGINE Bus STOPPED 2024-09-13T00:36:51.537 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[53580]: [13/Sep/2024:00:36:51] ENGINE Bus STARTING 2024-09-13T00:36:51.537 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[53580]: debug 2024-09-13T00:36:51.408+0000 7f02fcc09640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:36:51.537 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[53580]: debug 2024-09-13T00:36:51.408+0000 7f02fcc09640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:36:51.537 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[53580]: debug 2024-09-13T00:36:51.408+0000 7f02fcc09640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:36:51.537 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[53580]: debug 2024-09-13T00:36:51.408+0000 7f02fcc09640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:36:51.537 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[53580]: debug 2024-09-13T00:36:51.408+0000 7f02fcc09640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:36:51.537 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[53580]: debug 2024-09-13T00:36:51.412+0000 7f02ffc0f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:36:51.537 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[53580]: debug 2024-09-13T00:36:51.412+0000 7f02ffc0f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:36:51.537 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[53580]: debug 2024-09-13T00:36:51.412+0000 7f02ffc0f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:36:51.537 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[53580]: debug 2024-09-13T00:36:51.412+0000 7f02ffc0f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:36:51.537 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[53580]: debug 2024-09-13T00:36:51.412+0000 7f02ffc0f640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:36:51.537 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[53580]: [13/Sep/2024:00:36:51] ENGINE Serving on http://:::9283 2024-09-13T00:36:51.538 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[53580]: [13/Sep/2024:00:36:51] ENGINE Bus STARTED 2024-09-13T00:36:51.538 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: cluster 2024-09-13T00:36:49.911713+0000 mgr.y (mgr.14397) 272 : cluster [DBG] pgmap v344: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-09-13T00:36:51.538 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.661011+0000 mon.a (mon.0) 990 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:51.538 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.673996+0000 mon.a (mon.0) 991 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:51.538 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.677279+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:36:51.538 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.679070+0000 mon.c (mon.2) 157 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:36:51.538 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.689962+0000 mon.a (mon.0) 992 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:51.539 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.754899+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:51.539 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.766996+0000 mon.c (mon.2) 159 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T00:36:51.539 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.767987+0000 mon.a (mon.0) 993 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T00:36:51.539 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: cluster 2024-09-13T00:36:50.769014+0000 mon.a (mon.0) 994 : cluster [INF] Activating manager daemon x 2024-09-13T00:36:51.539 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: cluster 2024-09-13T00:36:50.782931+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2024-09-13T00:36:51.539 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.784672+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:51.539 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.784958+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T00:36:51.539 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: cluster 2024-09-13T00:36:50.785052+0000 mon.a (mon.0) 997 : cluster [DBG] mgrmap e22: x(active, starting, since 0.016043s) 2024-09-13T00:36:51.539 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.785167+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:51.539 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.785784+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:51.539 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.786406+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi077.bigbbf"}]: dispatch 2024-09-13T00:36:51.539 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.787238+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi088.krmowh"}]: dispatch 2024-09-13T00:36:51.539 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.787904+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T00:36:51.540 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.788572+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:36:51.540 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.789366+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:36:51.540 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.790672+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:36:51.540 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.791426+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:36:51.540 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.792121+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:36:51.540 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.792501+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:36:51.540 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.792922+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:36:51.540 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.793214+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:36:51.540 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.793703+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T00:36:51.540 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.794035+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T00:36:51.540 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:50.795059+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T00:36:51.540 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: cluster 2024-09-13T00:36:51.303618+0000 mon.a (mon.0) 998 : cluster [INF] Manager daemon x is now available 2024-09-13T00:36:51.540 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:51.324933+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:51.541 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:51.354439+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:51.541 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:51.366852+0000 mon.b (mon.1) 53 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:36:51.541 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:51.368848+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:36:51.541 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:51.401212+0000 mon.b (mon.1) 54 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T00:36:51.541 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:51 smithi088 bash[24895]: audit 2024-09-13T00:36:51.403059+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T00:36:51.693 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: cluster 2024-09-13T00:36:49.911713+0000 mgr.y (mgr.14397) 272 : cluster [DBG] pgmap v344: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-09-13T00:36:51.693 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.661011+0000 mon.a (mon.0) 990 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:51.693 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.673996+0000 mon.a (mon.0) 991 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:51.693 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.677279+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:36:51.693 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.679070+0000 mon.c (mon.2) 157 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:36:51.693 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.689962+0000 mon.a (mon.0) 992 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:51.693 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.754899+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:51.693 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.766996+0000 mon.c (mon.2) 159 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T00:36:51.693 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.767987+0000 mon.a (mon.0) 993 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T00:36:51.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: cluster 2024-09-13T00:36:50.769014+0000 mon.a (mon.0) 994 : cluster [INF] Activating manager daemon x 2024-09-13T00:36:51.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: cluster 2024-09-13T00:36:50.782931+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2024-09-13T00:36:51.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.784672+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:51.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.784958+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T00:36:51.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: cluster 2024-09-13T00:36:50.785052+0000 mon.a (mon.0) 997 : cluster [DBG] mgrmap e22: x(active, starting, since 0.016043s) 2024-09-13T00:36:51.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.785167+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:51.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.785784+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:51.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.786406+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi077.bigbbf"}]: dispatch 2024-09-13T00:36:51.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.787238+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi088.krmowh"}]: dispatch 2024-09-13T00:36:51.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.787904+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T00:36:51.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.788572+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:36:51.695 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: cluster 2024-09-13T00:36:49.911713+0000 mgr.y (mgr.14397) 272 : cluster [DBG] pgmap v344: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-09-13T00:36:51.695 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.661011+0000 mon.a (mon.0) 990 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:51.695 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.673996+0000 mon.a (mon.0) 991 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:51.695 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.677279+0000 mon.c (mon.2) 156 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:36:51.695 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.679070+0000 mon.c (mon.2) 157 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:36:51.695 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.689962+0000 mon.a (mon.0) 992 : audit [INF] from='mgr.14397 ' entity='mgr.y' 2024-09-13T00:36:51.695 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.754899+0000 mon.c (mon.2) 158 : audit [DBG] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:51.695 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.766996+0000 mon.c (mon.2) 159 : audit [INF] from='mgr.14397 172.21.15.77:0/1126921948' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T00:36:51.695 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.767987+0000 mon.a (mon.0) 993 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-13T00:36:51.696 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: cluster 2024-09-13T00:36:50.769014+0000 mon.a (mon.0) 994 : cluster [INF] Activating manager daemon x 2024-09-13T00:36:51.696 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: cluster 2024-09-13T00:36:50.782931+0000 mon.a (mon.0) 995 : cluster [DBG] osdmap e173: 8 total, 8 up, 8 in 2024-09-13T00:36:51.696 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.784672+0000 mon.b (mon.1) 34 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:36:51.696 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.784958+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.14397 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-13T00:36:51.696 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: cluster 2024-09-13T00:36:50.785052+0000 mon.a (mon.0) 997 : cluster [DBG] mgrmap e22: x(active, starting, since 0.016043s) 2024-09-13T00:36:51.696 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.785167+0000 mon.b (mon.1) 35 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:36:51.696 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.785784+0000 mon.b (mon.1) 36 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:36:51.696 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.786406+0000 mon.b (mon.1) 37 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi077.bigbbf"}]: dispatch 2024-09-13T00:36:51.696 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.787238+0000 mon.b (mon.1) 38 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi088.krmowh"}]: dispatch 2024-09-13T00:36:51.696 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.787904+0000 mon.b (mon.1) 39 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T00:36:51.696 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.788572+0000 mon.b (mon.1) 40 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:36:51.697 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.789366+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:36:51.697 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.790672+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:36:51.697 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.791426+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:36:51.697 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.792121+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:36:51.697 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.792501+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:36:51.697 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.792922+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:36:51.697 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.793214+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:36:51.697 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.793703+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T00:36:51.697 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.794035+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T00:36:51.698 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:50.795059+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T00:36:51.698 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: cluster 2024-09-13T00:36:51.303618+0000 mon.a (mon.0) 998 : cluster [INF] Manager daemon x is now available 2024-09-13T00:36:51.698 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:51.324933+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:51.698 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:51.354439+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:51.698 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:51.366852+0000 mon.b (mon.1) 53 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:36:51.698 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:51.368848+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:36:51.698 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:51.401212+0000 mon.b (mon.1) 54 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T00:36:51.698 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[32040]: audit 2024-09-13T00:36:51.403059+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T00:36:51.698 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.789366+0000 mon.b (mon.1) 41 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:36:51.698 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.790672+0000 mon.b (mon.1) 42 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:36:51.699 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.791426+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:36:51.699 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.792121+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:36:51.699 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.792501+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:36:51.699 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.792922+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:36:51.699 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.793214+0000 mon.b (mon.1) 47 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:36:51.699 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.793703+0000 mon.b (mon.1) 48 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T00:36:51.699 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.794035+0000 mon.b (mon.1) 49 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T00:36:51.699 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:50.795059+0000 mon.b (mon.1) 50 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T00:36:51.699 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: cluster 2024-09-13T00:36:51.303618+0000 mon.a (mon.0) 998 : cluster [INF] Manager daemon x is now available 2024-09-13T00:36:51.699 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:51.324933+0000 mon.b (mon.1) 51 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:36:51.699 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:51.354439+0000 mon.b (mon.1) 52 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:36:51.699 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:51.366852+0000 mon.b (mon.1) 53 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:36:51.700 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:51.368848+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:36:51.700 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:51.401212+0000 mon.b (mon.1) 54 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T00:36:51.700 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:51 smithi077 bash[22596]: audit 2024-09-13T00:36:51.403059+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-13T00:36:52.329 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 0 2024-09-13T00:36:52.442 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:36:52.525 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[22845]: debug 2024-09-13T00:36:52.265+0000 7f98f89b3ec0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T00:36:52.525 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[22845]: debug 2024-09-13T00:36:52.337+0000 7f98f89b3ec0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T00:36:52.584 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:36:53.026 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[22845]: debug 2024-09-13T00:36:52.549+0000 7f98f89b3ec0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T00:36:53.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[32040]: cluster 2024-09-13T00:36:51.777580+0000 mon.a (mon.0) 1001 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2024-09-13T00:36:53.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[32040]: audit 2024-09-13T00:36:51.820945+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:53.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[32040]: audit 2024-09-13T00:36:51.826544+0000 mon.a (mon.0) 1003 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:53.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[32040]: cephadm 2024-09-13T00:36:51.841623+0000 mgr.x (mgr.24490) 1 : cephadm [INF] Deploying cephadm binary to smithi088 2024-09-13T00:36:53.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[32040]: cephadm 2024-09-13T00:36:51.859675+0000 mgr.x (mgr.24490) 2 : cephadm [INF] Deploying cephadm binary to smithi077 2024-09-13T00:36:53.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[32040]: cluster 2024-09-13T00:36:52.143055+0000 mon.a (mon.0) 1004 : cluster [DBG] mgrmap e23: x(active, since 1.37404s) 2024-09-13T00:36:53.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[32040]: cluster 2024-09-13T00:36:52.165985+0000 mgr.x (mgr.24490) 3 : cluster [DBG] pgmap v4: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:36:53.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[32040]: audit 2024-09-13T00:36:52.457503+0000 mon.b (mon.1) 55 : audit [DBG] from='client.? 172.21.15.77:0/1312603032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:53.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[22596]: cluster 2024-09-13T00:36:51.777580+0000 mon.a (mon.0) 1001 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2024-09-13T00:36:53.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[22596]: audit 2024-09-13T00:36:51.820945+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:53.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[22596]: audit 2024-09-13T00:36:51.826544+0000 mon.a (mon.0) 1003 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:53.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[22596]: cephadm 2024-09-13T00:36:51.841623+0000 mgr.x (mgr.24490) 1 : cephadm [INF] Deploying cephadm binary to smithi088 2024-09-13T00:36:53.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[22596]: cephadm 2024-09-13T00:36:51.859675+0000 mgr.x (mgr.24490) 2 : cephadm [INF] Deploying cephadm binary to smithi077 2024-09-13T00:36:53.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[22596]: cluster 2024-09-13T00:36:52.143055+0000 mon.a (mon.0) 1004 : cluster [DBG] mgrmap e23: x(active, since 1.37404s) 2024-09-13T00:36:53.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[22596]: cluster 2024-09-13T00:36:52.165985+0000 mgr.x (mgr.24490) 3 : cluster [DBG] pgmap v4: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:36:53.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:52 smithi077 bash[22596]: audit 2024-09-13T00:36:52.457503+0000 mon.b (mon.1) 55 : audit [DBG] from='client.? 172.21.15.77:0/1312603032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:53.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:52 smithi088 bash[24895]: cluster 2024-09-13T00:36:51.777580+0000 mon.a (mon.0) 1001 : cluster [DBG] osdmap e174: 8 total, 8 up, 8 in 2024-09-13T00:36:53.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:52 smithi088 bash[24895]: audit 2024-09-13T00:36:51.820945+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:53.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:52 smithi088 bash[24895]: audit 2024-09-13T00:36:51.826544+0000 mon.a (mon.0) 1003 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:53.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:52 smithi088 bash[24895]: cephadm 2024-09-13T00:36:51.841623+0000 mgr.x (mgr.24490) 1 : cephadm [INF] Deploying cephadm binary to smithi088 2024-09-13T00:36:53.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:52 smithi088 bash[24895]: cephadm 2024-09-13T00:36:51.859675+0000 mgr.x (mgr.24490) 2 : cephadm [INF] Deploying cephadm binary to smithi077 2024-09-13T00:36:53.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:52 smithi088 bash[24895]: cluster 2024-09-13T00:36:52.143055+0000 mon.a (mon.0) 1004 : cluster [DBG] mgrmap e23: x(active, since 1.37404s) 2024-09-13T00:36:53.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:52 smithi088 bash[24895]: cluster 2024-09-13T00:36:52.165985+0000 mgr.x (mgr.24490) 3 : cluster [DBG] pgmap v4: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:36:53.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:52 smithi088 bash[24895]: audit 2024-09-13T00:36:52.457503+0000 mon.b (mon.1) 55 : audit [DBG] from='client.? 172.21.15.77:0/1312603032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:53.519 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:53 smithi077 bash[22845]: debug 2024-09-13T00:36:53.225+0000 7f98f89b3ec0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T00:36:53.519 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:53 smithi077 bash[22845]: debug 2024-09-13T00:36:53.293+0000 7f98f89b3ec0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T00:36:53.519 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:53 smithi077 bash[22845]: debug 2024-09-13T00:36:53.365+0000 7f98f89b3ec0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T00:36:53.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: debug 2024-09-13T00:36:53.220+0000 7f02e3bd7640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:00:36:53] ENGINE Error in HTTPServer.serve 2024-09-13T00:36:53.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: Traceback (most recent call last): 2024-09-13T00:36:53.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:36:53.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: self._connections.run(self.expiration_interval) 2024-09-13T00:36:53.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:36:53.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: self._run(expiration_interval) 2024-09-13T00:36:53.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:36:53.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:36:53.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:36:53.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:36:53.602 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:36:53.602 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: s = self.context.wrap_socket( 2024-09-13T00:36:53.602 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:36:53.602 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: return self.sslsocket_class._create( 2024-09-13T00:36:53.602 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:36:53.602 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: self.do_handshake() 2024-09-13T00:36:53.602 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:36:53.602 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: self._sslobj.do_handshake() 2024-09-13T00:36:53.602 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:36:53 smithi088 bash[53580]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:36:53.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:53 smithi077 bash[22845]: debug 2024-09-13T00:36:53.521+0000 7f98f89b3ec0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T00:36:53.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:53 smithi077 bash[22845]: debug 2024-09-13T00:36:53.597+0000 7f98f89b3ec0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T00:36:53.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:53 smithi077 bash[22845]: debug 2024-09-13T00:36:53.661+0000 7f98f89b3ec0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T00:36:54.147 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:53 smithi077 bash[22845]: debug 2024-09-13T00:36:53.893+0000 7f98f89b3ec0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T00:36:54.147 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:53 smithi077 bash[22845]: debug 2024-09-13T00:36:53.961+0000 7f98f89b3ec0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T00:36:54.147 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22845]: debug 2024-09-13T00:36:54.125+0000 7f98f89b3ec0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T00:36:54.456 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22845]: debug 2024-09-13T00:36:54.213+0000 7f98f89b3ec0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T00:36:54.457 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: cluster 2024-09-13T00:36:52.792695+0000 mgr.x (mgr.24490) 4 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:36:54.457 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: cephadm 2024-09-13T00:36:52.998796+0000 mgr.x (mgr.24490) 5 : cephadm [INF] [13/Sep/2024:00:36:52] ENGINE Bus STARTING 2024-09-13T00:36:54.457 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: cephadm 2024-09-13T00:36:53.101357+0000 mgr.x (mgr.24490) 6 : cephadm [INF] [13/Sep/2024:00:36:53] ENGINE Serving on http://172.21.15.88:8765 2024-09-13T00:36:54.457 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: cluster 2024-09-13T00:36:53.152345+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2024-09-13T00:36:54.457 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: cephadm 2024-09-13T00:36:53.221147+0000 mgr.x (mgr.24490) 7 : cephadm [INF] [13/Sep/2024:00:36:53] ENGINE Serving on https://172.21.15.88:7150 2024-09-13T00:36:54.457 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: cephadm 2024-09-13T00:36:53.221290+0000 mgr.x (mgr.24490) 8 : cephadm [INF] [13/Sep/2024:00:36:53] ENGINE Bus STARTED 2024-09-13T00:36:54.457 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: cephadm 2024-09-13T00:36:53.223266+0000 mgr.x (mgr.24490) 9 : cephadm [ERR] [13/Sep/2024:00:36:53] ENGINE Error in HTTPServer.serve 2024-09-13T00:36:54.458 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: Traceback (most recent call last): 2024-09-13T00:36:54.458 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:36:54.458 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: self._connections.run(self.expiration_interval) 2024-09-13T00:36:54.458 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:36:54.458 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: self._run(expiration_interval) 2024-09-13T00:36:54.458 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:36:54.458 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:36:54.458 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:36:54.458 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:36:54.458 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:36:54.458 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: s = self.context.wrap_socket( 2024-09-13T00:36:54.458 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:36:54.458 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: return self.sslsocket_class._create( 2024-09-13T00:36:54.459 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:36:54.459 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: self.do_handshake() 2024-09-13T00:36:54.459 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:36:54.459 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: self._sslobj.do_handshake() 2024-09-13T00:36:54.459 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[32040]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:36:54.459 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: cluster 2024-09-13T00:36:52.792695+0000 mgr.x (mgr.24490) 4 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:36:54.459 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: cephadm 2024-09-13T00:36:52.998796+0000 mgr.x (mgr.24490) 5 : cephadm [INF] [13/Sep/2024:00:36:52] ENGINE Bus STARTING 2024-09-13T00:36:54.459 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: cephadm 2024-09-13T00:36:53.101357+0000 mgr.x (mgr.24490) 6 : cephadm [INF] [13/Sep/2024:00:36:53] ENGINE Serving on http://172.21.15.88:8765 2024-09-13T00:36:54.459 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: cluster 2024-09-13T00:36:53.152345+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2024-09-13T00:36:54.459 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: cephadm 2024-09-13T00:36:53.221147+0000 mgr.x (mgr.24490) 7 : cephadm [INF] [13/Sep/2024:00:36:53] ENGINE Serving on https://172.21.15.88:7150 2024-09-13T00:36:54.459 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: cephadm 2024-09-13T00:36:53.221290+0000 mgr.x (mgr.24490) 8 : cephadm [INF] [13/Sep/2024:00:36:53] ENGINE Bus STARTED 2024-09-13T00:36:54.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: cephadm 2024-09-13T00:36:53.223266+0000 mgr.x (mgr.24490) 9 : cephadm [ERR] [13/Sep/2024:00:36:53] ENGINE Error in HTTPServer.serve 2024-09-13T00:36:54.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: Traceback (most recent call last): 2024-09-13T00:36:54.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:36:54.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: self._connections.run(self.expiration_interval) 2024-09-13T00:36:54.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:36:54.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: self._run(expiration_interval) 2024-09-13T00:36:54.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:36:54.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:36:54.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:36:54.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:36:54.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:36:54.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: s = self.context.wrap_socket( 2024-09-13T00:36:54.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:36:54.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: return self.sslsocket_class._create( 2024-09-13T00:36:54.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:36:54.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: self.do_handshake() 2024-09-13T00:36:54.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:36:54.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: self._sslobj.do_handshake() 2024-09-13T00:36:54.461 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22596]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:36:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: cluster 2024-09-13T00:36:52.792695+0000 mgr.x (mgr.24490) 4 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:36:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: cephadm 2024-09-13T00:36:52.998796+0000 mgr.x (mgr.24490) 5 : cephadm [INF] [13/Sep/2024:00:36:52] ENGINE Bus STARTING 2024-09-13T00:36:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: cephadm 2024-09-13T00:36:53.101357+0000 mgr.x (mgr.24490) 6 : cephadm [INF] [13/Sep/2024:00:36:53] ENGINE Serving on http://172.21.15.88:8765 2024-09-13T00:36:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: cluster 2024-09-13T00:36:53.152345+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e175: 8 total, 8 up, 8 in 2024-09-13T00:36:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: cephadm 2024-09-13T00:36:53.221147+0000 mgr.x (mgr.24490) 7 : cephadm [INF] [13/Sep/2024:00:36:53] ENGINE Serving on https://172.21.15.88:7150 2024-09-13T00:36:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: cephadm 2024-09-13T00:36:53.221290+0000 mgr.x (mgr.24490) 8 : cephadm [INF] [13/Sep/2024:00:36:53] ENGINE Bus STARTED 2024-09-13T00:36:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: cephadm 2024-09-13T00:36:53.223266+0000 mgr.x (mgr.24490) 9 : cephadm [ERR] [13/Sep/2024:00:36:53] ENGINE Error in HTTPServer.serve 2024-09-13T00:36:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: Traceback (most recent call last): 2024-09-13T00:36:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:36:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: self._connections.run(self.expiration_interval) 2024-09-13T00:36:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:36:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: self._run(expiration_interval) 2024-09-13T00:36:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:36:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:36:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:36:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:36:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:36:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: s = self.context.wrap_socket( 2024-09-13T00:36:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:36:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: return self.sslsocket_class._create( 2024-09-13T00:36:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:36:54.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: self.do_handshake() 2024-09-13T00:36:54.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:36:54.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: self._sslobj.do_handshake() 2024-09-13T00:36:54.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:54 smithi088 bash[24895]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:36:54.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22845]: debug 2024-09-13T00:36:54.457+0000 7f98f89b3ec0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:36:54.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22845]: debug 2024-09-13T00:36:54.625+0000 7f98f89b3ec0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T00:36:54.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22845]: debug 2024-09-13T00:36:54.705+0000 7f98f89b3ec0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T00:36:55.166 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:54 smithi077 bash[22845]: debug 2024-09-13T00:36:54.857+0000 7f98f89b3ec0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:36:55.523 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.SnapsShrinkToZeroAfterPrepare (12148 ms) 2024-09-13T00:36:55.523 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare 2024-09-13T00:36:55.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:55 smithi077 bash[22596]: cluster 2024-09-13T00:36:54.173260+0000 mon.a (mon.0) 1006 : cluster [DBG] mgrmap e24: x(active, since 3s) 2024-09-13T00:36:55.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:55 smithi077 bash[22596]: cluster 2024-09-13T00:36:54.173424+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2024-09-13T00:36:55.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:55 smithi077 bash[22596]: audit 2024-09-13T00:36:54.791018+0000 mon.a (mon.0) 1008 : audit [DBG] from='client.? 172.21.15.77:0/282829462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:55.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:55 smithi077 bash[32040]: cluster 2024-09-13T00:36:54.173260+0000 mon.a (mon.0) 1006 : cluster [DBG] mgrmap e24: x(active, since 3s) 2024-09-13T00:36:55.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:55 smithi077 bash[32040]: cluster 2024-09-13T00:36:54.173424+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2024-09-13T00:36:55.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:55 smithi077 bash[32040]: audit 2024-09-13T00:36:54.791018+0000 mon.a (mon.0) 1008 : audit [DBG] from='client.? 172.21.15.77:0/282829462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:55.588 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:36:55.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:55 smithi088 bash[24895]: cluster 2024-09-13T00:36:54.173260+0000 mon.a (mon.0) 1006 : cluster [DBG] mgrmap e24: x(active, since 3s) 2024-09-13T00:36:55.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:55 smithi088 bash[24895]: cluster 2024-09-13T00:36:54.173424+0000 mon.a (mon.0) 1007 : cluster [DBG] osdmap e176: 8 total, 8 up, 8 in 2024-09-13T00:36:55.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:55 smithi088 bash[24895]: audit 2024-09-13T00:36:54.791018+0000 mon.a (mon.0) 1008 : audit [DBG] from='client.? 172.21.15.77:0/282829462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:55.611 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:36:55.714 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:36:56.026 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:55 smithi077 bash[22845]: debug 2024-09-13T00:36:55.757+0000 7f98f89b3ec0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T00:36:56.026 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:55 smithi077 bash[22845]: debug 2024-09-13T00:36:55.837+0000 7f98f89b3ec0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T00:36:56.434 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:56 smithi077 bash[22845]: debug 2024-09-13T00:36:56.045+0000 7f98f89b3ec0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T00:36:56.434 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:56 smithi077 bash[22845]: debug 2024-09-13T00:36:56.117+0000 7f98f89b3ec0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T00:36:56.434 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:56 smithi077 bash[22596]: cluster 2024-09-13T00:36:54.793363+0000 mgr.x (mgr.24490) 10 : cluster [DBG] pgmap v8: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:36:56.434 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:56 smithi077 bash[22596]: cluster 2024-09-13T00:36:55.178281+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2024-09-13T00:36:56.434 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:56 smithi077 bash[32040]: cluster 2024-09-13T00:36:54.793363+0000 mgr.x (mgr.24490) 10 : cluster [DBG] pgmap v8: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:36:56.435 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:56 smithi077 bash[32040]: cluster 2024-09-13T00:36:55.178281+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2024-09-13T00:36:56.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:56 smithi088 bash[24895]: cluster 2024-09-13T00:36:54.793363+0000 mgr.x (mgr.24490) 10 : cluster [DBG] pgmap v8: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:36:56.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:56 smithi088 bash[24895]: cluster 2024-09-13T00:36:55.178281+0000 mon.a (mon.0) 1009 : cluster [DBG] osdmap e177: 8 total, 8 up, 8 in 2024-09-13T00:36:56.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:56 smithi077 bash[22845]: debug 2024-09-13T00:36:56.437+0000 7f98f89b3ec0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T00:36:56.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:56 smithi077 bash[22845]: debug 2024-09-13T00:36:56.513+0000 7f98f89b3ec0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T00:36:56.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:56 smithi077 bash[22845]: [13/Sep/2024:00:36:56] ENGINE Bus STARTING 2024-09-13T00:36:56.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:56 smithi077 bash[22845]: CherryPy Checker: 2024-09-13T00:36:56.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:56 smithi077 bash[22845]: The Application mounted at '' has an empty config. 2024-09-13T00:36:56.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:56 smithi077 bash[22845]: [13/Sep/2024:00:36:56] ENGINE Serving on http://:::9283 2024-09-13T00:36:56.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:36:56 smithi077 bash[22845]: [13/Sep/2024:00:36:56] ENGINE Bus STARTED 2024-09-13T00:36:57.233 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 8388609~8388609 2024-09-13T00:36:57.266 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 0~1000 2024-09-13T00:36:57.266 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 1010~1000 2024-09-13T00:36:57.266 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap2 2024-09-13T00:36:57.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:57 smithi077 bash[22596]: cluster 2024-09-13T00:36:56.177797+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2024-09-13T00:36:57.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:57 smithi077 bash[22596]: cluster 2024-09-13T00:36:56.537531+0000 mon.a (mon.0) 1011 : cluster [DBG] Standby manager daemon y started 2024-09-13T00:36:57.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:57 smithi077 bash[22596]: audit 2024-09-13T00:36:56.540627+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.? 172.21.15.77:0/3948117627' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T00:36:57.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:57 smithi077 bash[22596]: audit 2024-09-13T00:36:56.542478+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.? 172.21.15.77:0/3948117627' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:36:57.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:57 smithi077 bash[22596]: audit 2024-09-13T00:36:56.544156+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.? 172.21.15.77:0/3948117627' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T00:36:57.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:57 smithi077 bash[22596]: audit 2024-09-13T00:36:56.544500+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.? 172.21.15.77:0/3948117627' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:36:57.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:57 smithi077 bash[22596]: audit 2024-09-13T00:36:57.137204+0000 mon.a (mon.0) 1012 : audit [DBG] from='client.? 172.21.15.77:0/78660848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:57.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:57 smithi077 bash[32040]: cluster 2024-09-13T00:36:56.177797+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2024-09-13T00:36:57.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:57 smithi077 bash[32040]: cluster 2024-09-13T00:36:56.537531+0000 mon.a (mon.0) 1011 : cluster [DBG] Standby manager daemon y started 2024-09-13T00:36:57.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:57 smithi077 bash[32040]: audit 2024-09-13T00:36:56.540627+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.? 172.21.15.77:0/3948117627' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T00:36:57.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:57 smithi077 bash[32040]: audit 2024-09-13T00:36:56.542478+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.? 172.21.15.77:0/3948117627' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:36:57.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:57 smithi077 bash[32040]: audit 2024-09-13T00:36:56.544156+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.? 172.21.15.77:0/3948117627' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T00:36:57.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:57 smithi077 bash[32040]: audit 2024-09-13T00:36:56.544500+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.? 172.21.15.77:0/3948117627' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:36:57.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:57 smithi077 bash[32040]: audit 2024-09-13T00:36:57.137204+0000 mon.a (mon.0) 1012 : audit [DBG] from='client.? 172.21.15.77:0/78660848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:57.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:57 smithi088 bash[24895]: cluster 2024-09-13T00:36:56.177797+0000 mon.a (mon.0) 1010 : cluster [DBG] osdmap e178: 8 total, 8 up, 8 in 2024-09-13T00:36:57.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:57 smithi088 bash[24895]: cluster 2024-09-13T00:36:56.537531+0000 mon.a (mon.0) 1011 : cluster [DBG] Standby manager daemon y started 2024-09-13T00:36:57.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:57 smithi088 bash[24895]: audit 2024-09-13T00:36:56.540627+0000 mon.c (mon.2) 160 : audit [DBG] from='mgr.? 172.21.15.77:0/3948117627' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T00:36:57.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:57 smithi088 bash[24895]: audit 2024-09-13T00:36:56.542478+0000 mon.c (mon.2) 161 : audit [DBG] from='mgr.? 172.21.15.77:0/3948117627' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:36:57.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:57 smithi088 bash[24895]: audit 2024-09-13T00:36:56.544156+0000 mon.c (mon.2) 162 : audit [DBG] from='mgr.? 172.21.15.77:0/3948117627' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T00:36:57.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:57 smithi088 bash[24895]: audit 2024-09-13T00:36:56.544500+0000 mon.c (mon.2) 163 : audit [DBG] from='mgr.? 172.21.15.77:0/3948117627' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:36:57.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:57 smithi088 bash[24895]: audit 2024-09-13T00:36:57.137204+0000 mon.a (mon.0) 1012 : audit [DBG] from='client.? 172.21.15.77:0/78660848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:36:58.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:58 smithi077 bash[22596]: cluster 2024-09-13T00:36:56.794249+0000 mgr.x (mgr.24490) 11 : cluster [DBG] pgmap v11: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:36:58.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:58 smithi077 bash[22596]: cluster 2024-09-13T00:36:57.195855+0000 mon.a (mon.0) 1013 : cluster [DBG] mgrmap e25: x(active, since 6s), standbys: y 2024-09-13T00:36:58.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:58 smithi077 bash[22596]: audit 2024-09-13T00:36:57.204569+0000 mon.b (mon.1) 56 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T00:36:58.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:58 smithi077 bash[22596]: cluster 2024-09-13T00:36:57.210284+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2024-09-13T00:36:58.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:58 smithi077 bash[22596]: audit 2024-09-13T00:36:57.908366+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:58.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:58 smithi077 bash[22596]: audit 2024-09-13T00:36:57.914357+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:58.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:58 smithi077 bash[32040]: cluster 2024-09-13T00:36:56.794249+0000 mgr.x (mgr.24490) 11 : cluster [DBG] pgmap v11: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:36:58.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:58 smithi077 bash[32040]: cluster 2024-09-13T00:36:57.195855+0000 mon.a (mon.0) 1013 : cluster [DBG] mgrmap e25: x(active, since 6s), standbys: y 2024-09-13T00:36:58.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:58 smithi077 bash[32040]: audit 2024-09-13T00:36:57.204569+0000 mon.b (mon.1) 56 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T00:36:58.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:58 smithi077 bash[32040]: cluster 2024-09-13T00:36:57.210284+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2024-09-13T00:36:58.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:58 smithi077 bash[32040]: audit 2024-09-13T00:36:57.908366+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:58.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:58 smithi077 bash[32040]: audit 2024-09-13T00:36:57.914357+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:58.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:58 smithi088 bash[24895]: cluster 2024-09-13T00:36:56.794249+0000 mgr.x (mgr.24490) 11 : cluster [DBG] pgmap v11: 137 pgs: 137 active+clean; 561 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:36:58.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:58 smithi088 bash[24895]: cluster 2024-09-13T00:36:57.195855+0000 mon.a (mon.0) 1013 : cluster [DBG] mgrmap e25: x(active, since 6s), standbys: y 2024-09-13T00:36:58.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:58 smithi088 bash[24895]: audit 2024-09-13T00:36:57.204569+0000 mon.b (mon.1) 56 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T00:36:58.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:58 smithi088 bash[24895]: cluster 2024-09-13T00:36:57.210284+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e179: 8 total, 8 up, 8 in 2024-09-13T00:36:58.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:58 smithi088 bash[24895]: audit 2024-09-13T00:36:57.908366+0000 mon.a (mon.0) 1015 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:58.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:58 smithi088 bash[24895]: audit 2024-09-13T00:36:57.914357+0000 mon.a (mon.0) 1016 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.250 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 8388609~1000 2024-09-13T00:36:59.267 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 16777228~1000 2024-09-13T00:36:59.267 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 167772160 2024-09-13T00:36:59.282 INFO:tasks.workunit.client.0.smithi077.stdout:write: 2 83886079~8388609 2024-09-13T00:36:59.317 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap3 2024-09-13T00:36:59.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: cluster 2024-09-13T00:36:58.219036+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2024-09-13T00:36:59.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: audit 2024-09-13T00:36:58.391196+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: audit 2024-09-13T00:36:58.402463+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: audit 2024-09-13T00:36:58.574599+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: audit 2024-09-13T00:36:58.585111+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: audit 2024-09-13T00:36:58.587658+0000 mon.b (mon.1) 57 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: audit 2024-09-13T00:36:58.589447+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: audit 2024-09-13T00:36:58.589639+0000 mon.b (mon.1) 58 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: audit 2024-09-13T00:36:58.591373+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: audit 2024-09-13T00:36:58.591405+0000 mon.a (mon.0) 1023 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: audit 2024-09-13T00:36:58.593143+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: audit 2024-09-13T00:36:58.593313+0000 mon.b (mon.1) 60 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: cephadm 2024-09-13T00:36:58.594859+0000 mgr.x (mgr.24490) 12 : cephadm [INF] Adjusting osd_memory_target on smithi077 to 2524M 2024-09-13T00:36:59.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: audit 2024-09-13T00:36:58.595083+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: audit 2024-09-13T00:36:58.603481+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: audit 2024-09-13T00:36:59.079507+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[22596]: audit 2024-09-13T00:36:59.091048+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: cluster 2024-09-13T00:36:58.219036+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2024-09-13T00:36:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: audit 2024-09-13T00:36:58.391196+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: audit 2024-09-13T00:36:58.402463+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: audit 2024-09-13T00:36:58.574599+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: audit 2024-09-13T00:36:58.585111+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: audit 2024-09-13T00:36:58.587658+0000 mon.b (mon.1) 57 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: audit 2024-09-13T00:36:58.589447+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: audit 2024-09-13T00:36:58.589639+0000 mon.b (mon.1) 58 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: audit 2024-09-13T00:36:58.591373+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: audit 2024-09-13T00:36:58.591405+0000 mon.a (mon.0) 1023 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: audit 2024-09-13T00:36:58.593143+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: audit 2024-09-13T00:36:58.593313+0000 mon.b (mon.1) 60 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: cephadm 2024-09-13T00:36:58.594859+0000 mgr.x (mgr.24490) 12 : cephadm [INF] Adjusting osd_memory_target on smithi077 to 2524M 2024-09-13T00:36:59.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: audit 2024-09-13T00:36:58.595083+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: audit 2024-09-13T00:36:58.603481+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: audit 2024-09-13T00:36:59.079507+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:36:59 smithi077 bash[32040]: audit 2024-09-13T00:36:59.091048+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: cluster 2024-09-13T00:36:58.219036+0000 mon.a (mon.0) 1017 : cluster [DBG] osdmap e180: 8 total, 8 up, 8 in 2024-09-13T00:36:59.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: audit 2024-09-13T00:36:58.391196+0000 mon.a (mon.0) 1018 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: audit 2024-09-13T00:36:58.402463+0000 mon.a (mon.0) 1019 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: audit 2024-09-13T00:36:58.574599+0000 mon.a (mon.0) 1020 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: audit 2024-09-13T00:36:58.585111+0000 mon.a (mon.0) 1021 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: audit 2024-09-13T00:36:58.587658+0000 mon.b (mon.1) 57 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: audit 2024-09-13T00:36:58.589447+0000 mon.a (mon.0) 1022 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: audit 2024-09-13T00:36:58.589639+0000 mon.b (mon.1) 58 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: audit 2024-09-13T00:36:58.591373+0000 mon.b (mon.1) 59 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: audit 2024-09-13T00:36:58.591405+0000 mon.a (mon.0) 1023 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: audit 2024-09-13T00:36:58.593143+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: audit 2024-09-13T00:36:58.593313+0000 mon.b (mon.1) 60 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: cephadm 2024-09-13T00:36:58.594859+0000 mgr.x (mgr.24490) 12 : cephadm [INF] Adjusting osd_memory_target on smithi077 to 2524M 2024-09-13T00:36:59.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: audit 2024-09-13T00:36:58.595083+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:36:59.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: audit 2024-09-13T00:36:58.603481+0000 mon.a (mon.0) 1026 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: audit 2024-09-13T00:36:59.079507+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:36:59.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:36:59 smithi088 bash[24895]: audit 2024-09-13T00:36:59.091048+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:00.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:00 smithi077 bash[22596]: cluster 2024-09-13T00:36:58.794819+0000 mgr.x (mgr.24490) 13 : cluster [DBG] pgmap v14: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 42 MiB/s rd, 10 MiB/s wr, 468 op/s 2024-09-13T00:37:00.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:00 smithi077 bash[22596]: cluster 2024-09-13T00:36:59.236235+0000 mon.a (mon.0) 1029 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2024-09-13T00:37:00.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:00 smithi077 bash[22596]: audit 2024-09-13T00:36:59.519320+0000 mon.a (mon.0) 1030 : audit [DBG] from='client.? 172.21.15.77:0/3600737965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:00.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:00 smithi077 bash[32040]: cluster 2024-09-13T00:36:58.794819+0000 mgr.x (mgr.24490) 13 : cluster [DBG] pgmap v14: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 42 MiB/s rd, 10 MiB/s wr, 468 op/s 2024-09-13T00:37:00.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:00 smithi077 bash[32040]: cluster 2024-09-13T00:36:59.236235+0000 mon.a (mon.0) 1029 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2024-09-13T00:37:00.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:00 smithi077 bash[32040]: audit 2024-09-13T00:36:59.519320+0000 mon.a (mon.0) 1030 : audit [DBG] from='client.? 172.21.15.77:0/3600737965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:00.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:00 smithi088 bash[24895]: cluster 2024-09-13T00:36:58.794819+0000 mgr.x (mgr.24490) 13 : cluster [DBG] pgmap v14: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 42 MiB/s rd, 10 MiB/s wr, 468 op/s 2024-09-13T00:37:00.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:00 smithi088 bash[24895]: cluster 2024-09-13T00:36:59.236235+0000 mon.a (mon.0) 1029 : cluster [DBG] osdmap e181: 8 total, 8 up, 8 in 2024-09-13T00:37:00.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:00 smithi088 bash[24895]: audit 2024-09-13T00:36:59.519320+0000 mon.a (mon.0) 1030 : audit [DBG] from='client.? 172.21.15.77:0/3600737965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:01.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:00 smithi088 bash[53580]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:37:00] "GET /metrics HTTP/1.1" 200 35770 "" "Prometheus/2.43.0" 2024-09-13T00:37:01.161 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:37:01.221 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 83886079~1 2024-09-13T00:37:01.241 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:37:01.526 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:01 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:37:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:37:01.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:01 smithi077 bash[22596]: cluster 2024-09-13T00:37:00.248532+0000 mon.a (mon.0) 1031 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2024-09-13T00:37:01.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:01 smithi077 bash[22596]: cluster 2024-09-13T00:37:01.143938+0000 mon.a (mon.0) 1032 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2024-09-13T00:37:01.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:01 smithi077 bash[32040]: cluster 2024-09-13T00:37:00.248532+0000 mon.a (mon.0) 1031 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2024-09-13T00:37:01.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:01 smithi077 bash[32040]: cluster 2024-09-13T00:37:01.143938+0000 mon.a (mon.0) 1032 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2024-09-13T00:37:01.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:01 smithi088 bash[24895]: cluster 2024-09-13T00:37:00.248532+0000 mon.a (mon.0) 1031 : cluster [DBG] osdmap e182: 8 total, 8 up, 8 in 2024-09-13T00:37:01.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:01 smithi088 bash[24895]: cluster 2024-09-13T00:37:01.143938+0000 mon.a (mon.0) 1032 : cluster [DBG] osdmap e183: 8 total, 8 up, 8 in 2024-09-13T00:37:02.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:02 smithi077 bash[22596]: cluster 2024-09-13T00:37:00.795618+0000 mgr.x (mgr.24490) 14 : cluster [DBG] pgmap v17: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 42 MiB/s rd, 10 MiB/s wr, 468 op/s 2024-09-13T00:37:02.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:02 smithi077 bash[22596]: audit 2024-09-13T00:37:01.879302+0000 mon.a (mon.0) 1033 : audit [DBG] from='client.? 172.21.15.77:0/1279447226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:02.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:02 smithi077 bash[32040]: cluster 2024-09-13T00:37:00.795618+0000 mgr.x (mgr.24490) 14 : cluster [DBG] pgmap v17: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 42 MiB/s rd, 10 MiB/s wr, 468 op/s 2024-09-13T00:37:02.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:02 smithi077 bash[32040]: audit 2024-09-13T00:37:01.879302+0000 mon.a (mon.0) 1033 : audit [DBG] from='client.? 172.21.15.77:0/1279447226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:02.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:02 smithi088 bash[24895]: cluster 2024-09-13T00:37:00.795618+0000 mgr.x (mgr.24490) 14 : cluster [DBG] pgmap v17: 137 pgs: 137 active+clean; 577 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 42 MiB/s rd, 10 MiB/s wr, 468 op/s 2024-09-13T00:37:02.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:02 smithi088 bash[24895]: audit 2024-09-13T00:37:01.879302+0000 mon.a (mon.0) 1033 : audit [DBG] from='client.? 172.21.15.77:0/1279447226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:03.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:03 smithi077 bash[22596]: cluster 2024-09-13T00:37:02.267924+0000 mon.a (mon.0) 1034 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2024-09-13T00:37:03.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:03 smithi077 bash[32040]: cluster 2024-09-13T00:37:02.267924+0000 mon.a (mon.0) 1034 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2024-09-13T00:37:03.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:03 smithi088 bash[24895]: cluster 2024-09-13T00:37:02.267924+0000 mon.a (mon.0) 1034 : cluster [DBG] osdmap e184: 8 total, 8 up, 8 in 2024-09-13T00:37:04.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:04 smithi088 bash[24895]: cluster 2024-09-13T00:37:02.796512+0000 mgr.x (mgr.24490) 15 : cluster [DBG] pgmap v20: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-13T00:37:04.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:04 smithi088 bash[24895]: cluster 2024-09-13T00:37:03.278236+0000 mon.a (mon.0) 1035 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2024-09-13T00:37:04.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:04 smithi088 bash[24895]: audit 2024-09-13T00:37:04.238395+0000 mon.a (mon.0) 1036 : audit [DBG] from='client.? 172.21.15.77:0/1439426626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:04.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:04 smithi077 bash[22596]: cluster 2024-09-13T00:37:02.796512+0000 mgr.x (mgr.24490) 15 : cluster [DBG] pgmap v20: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-13T00:37:04.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:04 smithi077 bash[22596]: cluster 2024-09-13T00:37:03.278236+0000 mon.a (mon.0) 1035 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2024-09-13T00:37:04.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:04 smithi077 bash[22596]: audit 2024-09-13T00:37:04.238395+0000 mon.a (mon.0) 1036 : audit [DBG] from='client.? 172.21.15.77:0/1439426626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:04.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:04 smithi077 bash[32040]: cluster 2024-09-13T00:37:02.796512+0000 mgr.x (mgr.24490) 15 : cluster [DBG] pgmap v20: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-13T00:37:04.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:04 smithi077 bash[32040]: cluster 2024-09-13T00:37:03.278236+0000 mon.a (mon.0) 1035 : cluster [DBG] osdmap e185: 8 total, 8 up, 8 in 2024-09-13T00:37:04.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:04 smithi077 bash[32040]: audit 2024-09-13T00:37:04.238395+0000 mon.a (mon.0) 1036 : audit [DBG] from='client.? 172.21.15.77:0/1439426626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:05.169 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 167772160 2024-09-13T00:37:05.196 INFO:tasks.workunit.client.0.smithi077.stdout:write: * 83886080~1000 2024-09-13T00:37:05.260 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:37:05.515 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:37:05.580 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:05 smithi088 bash[24895]: cluster 2024-09-13T00:37:04.284801+0000 mon.a (mon.0) 1037 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2024-09-13T00:37:05.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:05 smithi077 bash[22596]: cluster 2024-09-13T00:37:04.284801+0000 mon.a (mon.0) 1037 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2024-09-13T00:37:05.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:05 smithi077 bash[32040]: cluster 2024-09-13T00:37:04.284801+0000 mon.a (mon.0) 1037 : cluster [DBG] osdmap e186: 8 total, 8 up, 8 in 2024-09-13T00:37:06.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:06 smithi088 bash[24895]: cluster 2024-09-13T00:37:04.797282+0000 mgr.x (mgr.24490) 16 : cluster [DBG] pgmap v23: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-13T00:37:06.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:06 smithi077 bash[22596]: cluster 2024-09-13T00:37:04.797282+0000 mgr.x (mgr.24490) 16 : cluster [DBG] pgmap v23: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-13T00:37:06.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:06 smithi077 bash[32040]: cluster 2024-09-13T00:37:04.797282+0000 mgr.x (mgr.24490) 16 : cluster [DBG] pgmap v23: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-13T00:37:07.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:07 smithi088 bash[24895]: cluster 2024-09-13T00:37:06.301526+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2024-09-13T00:37:07.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:07 smithi088 bash[24895]: audit 2024-09-13T00:37:06.354128+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:07.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:07 smithi088 bash[24895]: audit 2024-09-13T00:37:06.605803+0000 mon.c (mon.2) 164 : audit [DBG] from='client.? 172.21.15.77:0/1632061387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:07.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:07 smithi077 bash[22596]: cluster 2024-09-13T00:37:06.301526+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2024-09-13T00:37:07.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:07 smithi077 bash[22596]: audit 2024-09-13T00:37:06.354128+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:07.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:07 smithi077 bash[22596]: audit 2024-09-13T00:37:06.605803+0000 mon.c (mon.2) 164 : audit [DBG] from='client.? 172.21.15.77:0/1632061387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:07.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:07 smithi077 bash[32040]: cluster 2024-09-13T00:37:06.301526+0000 mon.a (mon.0) 1038 : cluster [DBG] osdmap e187: 8 total, 8 up, 8 in 2024-09-13T00:37:07.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:07 smithi077 bash[32040]: audit 2024-09-13T00:37:06.354128+0000 mon.b (mon.1) 61 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:07.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:07 smithi077 bash[32040]: audit 2024-09-13T00:37:06.605803+0000 mon.c (mon.2) 164 : audit [DBG] from='client.? 172.21.15.77:0/1632061387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:08.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:08 smithi088 bash[24895]: cluster 2024-09-13T00:37:06.798081+0000 mgr.x (mgr.24490) 17 : cluster [DBG] pgmap v25: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:08.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:08 smithi088 bash[24895]: cluster 2024-09-13T00:37:07.316103+0000 mon.a (mon.0) 1039 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2024-09-13T00:37:08.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:08 smithi077 bash[22596]: cluster 2024-09-13T00:37:06.798081+0000 mgr.x (mgr.24490) 17 : cluster [DBG] pgmap v25: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:08.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:08 smithi077 bash[22596]: cluster 2024-09-13T00:37:07.316103+0000 mon.a (mon.0) 1039 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2024-09-13T00:37:08.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:08 smithi077 bash[32040]: cluster 2024-09-13T00:37:06.798081+0000 mgr.x (mgr.24490) 17 : cluster [DBG] pgmap v25: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:08.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:08 smithi077 bash[32040]: cluster 2024-09-13T00:37:07.316103+0000 mon.a (mon.0) 1039 : cluster [DBG] osdmap e188: 8 total, 8 up, 8 in 2024-09-13T00:37:08.903 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare (13381 ms) 2024-09-13T00:37:08.904 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare2 2024-09-13T00:37:08.979 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:37:09.002 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 41943040~10 2024-09-13T00:37:09.015 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:37:09.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:09 smithi088 bash[24895]: cluster 2024-09-13T00:37:08.327337+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2024-09-13T00:37:09.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:09 smithi088 bash[24895]: audit 2024-09-13T00:37:08.954609+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:09.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:09 smithi088 bash[24895]: audit 2024-09-13T00:37:08.968173+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:09.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:09 smithi088 bash[24895]: audit 2024-09-13T00:37:08.970439+0000 mon.b (mon.1) 62 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:09 smithi088 bash[24895]: audit 2024-09-13T00:37:08.972282+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:09 smithi088 bash[24895]: audit 2024-09-13T00:37:08.972724+0000 mon.b (mon.1) 63 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:09 smithi088 bash[24895]: audit 2024-09-13T00:37:08.974651+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:09 smithi088 bash[24895]: audit 2024-09-13T00:37:08.975303+0000 mon.b (mon.1) 64 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:09 smithi088 bash[24895]: audit 2024-09-13T00:37:08.977158+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:09 smithi088 bash[24895]: audit 2024-09-13T00:37:08.977570+0000 mon.b (mon.1) 65 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:09 smithi088 bash[24895]: audit 2024-09-13T00:37:08.979400+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:09 smithi088 bash[24895]: audit 2024-09-13T00:37:08.990062+0000 mon.a (mon.0) 1047 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:09.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:09 smithi088 bash[24895]: audit 2024-09-13T00:37:08.993838+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:37:09.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:09 smithi088 bash[24895]: audit 2024-09-13T00:37:08.995599+0000 mon.b (mon.1) 67 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:37:09.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:09 smithi088 bash[24895]: audit 2024-09-13T00:37:08.997982+0000 mon.a (mon.0) 1048 : audit [DBG] from='client.? 172.21.15.77:0/1809283975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:09.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[22596]: cluster 2024-09-13T00:37:08.327337+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2024-09-13T00:37:09.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[22596]: audit 2024-09-13T00:37:08.954609+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:09.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[22596]: audit 2024-09-13T00:37:08.968173+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:09.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[22596]: audit 2024-09-13T00:37:08.970439+0000 mon.b (mon.1) 62 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[22596]: audit 2024-09-13T00:37:08.972282+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[22596]: audit 2024-09-13T00:37:08.972724+0000 mon.b (mon.1) 63 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[22596]: audit 2024-09-13T00:37:08.974651+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[22596]: audit 2024-09-13T00:37:08.975303+0000 mon.b (mon.1) 64 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[22596]: audit 2024-09-13T00:37:08.977158+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[22596]: audit 2024-09-13T00:37:08.977570+0000 mon.b (mon.1) 65 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[22596]: audit 2024-09-13T00:37:08.979400+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[22596]: audit 2024-09-13T00:37:08.990062+0000 mon.a (mon.0) 1047 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:09.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[22596]: audit 2024-09-13T00:37:08.993838+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:37:09.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[22596]: audit 2024-09-13T00:37:08.995599+0000 mon.b (mon.1) 67 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:37:09.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[22596]: audit 2024-09-13T00:37:08.997982+0000 mon.a (mon.0) 1048 : audit [DBG] from='client.? 172.21.15.77:0/1809283975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:09.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[32040]: cluster 2024-09-13T00:37:08.327337+0000 mon.a (mon.0) 1040 : cluster [DBG] osdmap e189: 8 total, 8 up, 8 in 2024-09-13T00:37:09.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[32040]: audit 2024-09-13T00:37:08.954609+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:09.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[32040]: audit 2024-09-13T00:37:08.968173+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:09.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[32040]: audit 2024-09-13T00:37:08.970439+0000 mon.b (mon.1) 62 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[32040]: audit 2024-09-13T00:37:08.972282+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[32040]: audit 2024-09-13T00:37:08.972724+0000 mon.b (mon.1) 63 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[32040]: audit 2024-09-13T00:37:08.974651+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[32040]: audit 2024-09-13T00:37:08.975303+0000 mon.b (mon.1) 64 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[32040]: audit 2024-09-13T00:37:08.977158+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[32040]: audit 2024-09-13T00:37:08.977570+0000 mon.b (mon.1) 65 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[32040]: audit 2024-09-13T00:37:08.979400+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-13T00:37:09.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[32040]: audit 2024-09-13T00:37:08.990062+0000 mon.a (mon.0) 1047 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:09.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[32040]: audit 2024-09-13T00:37:08.993838+0000 mon.b (mon.1) 66 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:37:09.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[32040]: audit 2024-09-13T00:37:08.995599+0000 mon.b (mon.1) 67 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:37:09.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:09 smithi077 bash[32040]: audit 2024-09-13T00:37:08.997982+0000 mon.a (mon.0) 1048 : audit [DBG] from='client.? 172.21.15.77:0/1809283975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:10.275 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:09 smithi077 systemd[1]: Stopping Ceph node-exporter.a for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:37:10.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: cluster 2024-09-13T00:37:08.798914+0000 mgr.x (mgr.24490) 18 : cluster [DBG] pgmap v28: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 5.3 MiB/s wr, 377 op/s 2024-09-13T00:37:10.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: cephadm 2024-09-13T00:37:08.979618+0000 mgr.x (mgr.24490) 19 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 2524M 2024-09-13T00:37:10.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: cephadm 2024-09-13T00:37:08.997613+0000 mgr.x (mgr.24490) 20 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.conf 2024-09-13T00:37:10.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: cephadm 2024-09-13T00:37:08.998083+0000 mgr.x (mgr.24490) 21 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:37:10.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: cephadm 2024-09-13T00:37:09.177826+0000 mgr.x (mgr.24490) 22 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:37:10.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: cephadm 2024-09-13T00:37:09.178106+0000 mgr.x (mgr.24490) 23 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:37:10.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: cephadm 2024-09-13T00:37:09.316128+0000 mgr.x (mgr.24490) 24 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:37:10.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: cephadm 2024-09-13T00:37:09.316294+0000 mgr.x (mgr.24490) 25 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:37:10.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: cephadm 2024-09-13T00:37:09.442452+0000 mgr.x (mgr.24490) 26 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:37:10.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: cephadm 2024-09-13T00:37:09.449683+0000 mgr.x (mgr.24490) 27 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:37:10.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: audit 2024-09-13T00:37:09.571086+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: audit 2024-09-13T00:37:09.580094+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: audit 2024-09-13T00:37:09.589149+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: audit 2024-09-13T00:37:09.599805+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: audit 2024-09-13T00:37:09.609787+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: audit 2024-09-13T00:37:09.653861+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: audit 2024-09-13T00:37:09.662698+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: audit 2024-09-13T00:37:09.671912+0000 mon.a (mon.0) 1056 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: audit 2024-09-13T00:37:09.681961+0000 mon.a (mon.0) 1057 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[22596]: cluster 2024-09-13T00:37:09.995697+0000 mon.a (mon.0) 1058 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2024-09-13T00:37:10.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: cluster 2024-09-13T00:37:08.798914+0000 mgr.x (mgr.24490) 18 : cluster [DBG] pgmap v28: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 5.3 MiB/s wr, 377 op/s 2024-09-13T00:37:10.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: cephadm 2024-09-13T00:37:08.979618+0000 mgr.x (mgr.24490) 19 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 2524M 2024-09-13T00:37:10.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: cephadm 2024-09-13T00:37:08.997613+0000 mgr.x (mgr.24490) 20 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.conf 2024-09-13T00:37:10.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: cephadm 2024-09-13T00:37:08.998083+0000 mgr.x (mgr.24490) 21 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:37:10.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: cephadm 2024-09-13T00:37:09.177826+0000 mgr.x (mgr.24490) 22 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:37:10.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: cephadm 2024-09-13T00:37:09.178106+0000 mgr.x (mgr.24490) 23 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:37:10.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: cephadm 2024-09-13T00:37:09.316128+0000 mgr.x (mgr.24490) 24 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:37:10.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: cephadm 2024-09-13T00:37:09.316294+0000 mgr.x (mgr.24490) 25 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:37:10.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: cephadm 2024-09-13T00:37:09.442452+0000 mgr.x (mgr.24490) 26 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:37:10.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: cephadm 2024-09-13T00:37:09.449683+0000 mgr.x (mgr.24490) 27 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:37:10.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: audit 2024-09-13T00:37:09.571086+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: audit 2024-09-13T00:37:09.580094+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: audit 2024-09-13T00:37:09.589149+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: audit 2024-09-13T00:37:09.599805+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: audit 2024-09-13T00:37:09.609787+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: audit 2024-09-13T00:37:09.653861+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: audit 2024-09-13T00:37:09.662698+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: audit 2024-09-13T00:37:09.671912+0000 mon.a (mon.0) 1056 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: audit 2024-09-13T00:37:09.681961+0000 mon.a (mon.0) 1057 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:10 smithi077 bash[32040]: cluster 2024-09-13T00:37:09.995697+0000 mon.a (mon.0) 1058 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2024-09-13T00:37:10.850 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[53580]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:37:10] "GET /metrics HTTP/1.1" 200 38825 "" "Prometheus/2.43.0" 2024-09-13T00:37:10.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: cluster 2024-09-13T00:37:08.798914+0000 mgr.x (mgr.24490) 18 : cluster [DBG] pgmap v28: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 5.3 MiB/s wr, 377 op/s 2024-09-13T00:37:10.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: cephadm 2024-09-13T00:37:08.979618+0000 mgr.x (mgr.24490) 19 : cephadm [INF] Adjusting osd_memory_target on smithi088 to 2524M 2024-09-13T00:37:10.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: cephadm 2024-09-13T00:37:08.997613+0000 mgr.x (mgr.24490) 20 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.conf 2024-09-13T00:37:10.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: cephadm 2024-09-13T00:37:08.998083+0000 mgr.x (mgr.24490) 21 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:37:10.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: cephadm 2024-09-13T00:37:09.177826+0000 mgr.x (mgr.24490) 22 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:37:10.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: cephadm 2024-09-13T00:37:09.178106+0000 mgr.x (mgr.24490) 23 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:37:10.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: cephadm 2024-09-13T00:37:09.316128+0000 mgr.x (mgr.24490) 24 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:37:10.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: cephadm 2024-09-13T00:37:09.316294+0000 mgr.x (mgr.24490) 25 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:37:10.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: cephadm 2024-09-13T00:37:09.442452+0000 mgr.x (mgr.24490) 26 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:37:10.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: cephadm 2024-09-13T00:37:09.449683+0000 mgr.x (mgr.24490) 27 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:37:10.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: audit 2024-09-13T00:37:09.571086+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: audit 2024-09-13T00:37:09.580094+0000 mon.a (mon.0) 1050 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: audit 2024-09-13T00:37:09.589149+0000 mon.a (mon.0) 1051 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: audit 2024-09-13T00:37:09.599805+0000 mon.a (mon.0) 1052 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: audit 2024-09-13T00:37:09.609787+0000 mon.a (mon.0) 1053 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: audit 2024-09-13T00:37:09.653861+0000 mon.a (mon.0) 1054 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: audit 2024-09-13T00:37:09.662698+0000 mon.a (mon.0) 1055 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: audit 2024-09-13T00:37:09.671912+0000 mon.a (mon.0) 1056 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: audit 2024-09-13T00:37:09.681961+0000 mon.a (mon.0) 1057 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:10.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:10 smithi088 bash[24895]: cluster 2024-09-13T00:37:09.995697+0000 mon.a (mon.0) 1058 : cluster [DBG] osdmap e190: 8 total, 8 up, 8 in 2024-09-13T00:37:11.017 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 41943040 2024-09-13T00:37:11.061 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:37:11.462 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[72035]: ceph-9218306c-7165-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T00:37:11.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:37:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:37:11.776 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:11 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T00:37:11.776 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[72119]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T00:37:11.776 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:11 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T00:37:11.776 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:11 smithi077 systemd[1]: Stopped Ceph node-exporter.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:37:11.776 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:11 smithi077 systemd[1]: Started Ceph node-exporter.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:37:11.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[22596]: cephadm 2024-09-13T00:37:09.682282+0000 mgr.x (mgr.24490) 28 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T00:37:11.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[22596]: cephadm 2024-09-13T00:37:09.682888+0000 mgr.x (mgr.24490) 29 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi077 2024-09-13T00:37:11.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[22596]: cluster 2024-09-13T00:37:10.998324+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2024-09-13T00:37:11.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[22596]: cluster 2024-09-13T00:37:11.142800+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2024-09-13T00:37:11.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[22596]: audit 2024-09-13T00:37:11.395719+0000 mon.c (mon.2) 165 : audit [DBG] from='client.? 172.21.15.77:0/2393778847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:11.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[22596]: audit 2024-09-13T00:37:11.557928+0000 mon.a (mon.0) 1061 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:11.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[22596]: audit 2024-09-13T00:37:11.568290+0000 mon.a (mon.0) 1062 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:11.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[32040]: cephadm 2024-09-13T00:37:09.682282+0000 mgr.x (mgr.24490) 28 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T00:37:11.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[32040]: cephadm 2024-09-13T00:37:09.682888+0000 mgr.x (mgr.24490) 29 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi077 2024-09-13T00:37:11.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[32040]: cluster 2024-09-13T00:37:10.998324+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2024-09-13T00:37:11.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[32040]: cluster 2024-09-13T00:37:11.142800+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2024-09-13T00:37:11.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[32040]: audit 2024-09-13T00:37:11.395719+0000 mon.c (mon.2) 165 : audit [DBG] from='client.? 172.21.15.77:0/2393778847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:11.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[32040]: audit 2024-09-13T00:37:11.557928+0000 mon.a (mon.0) 1061 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:11.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:11 smithi077 bash[32040]: audit 2024-09-13T00:37:11.568290+0000 mon.a (mon.0) 1062 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:11.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:11 smithi088 bash[24895]: cephadm 2024-09-13T00:37:09.682282+0000 mgr.x (mgr.24490) 28 : cephadm [INF] Reconfiguring node-exporter.a (dependencies changed)... 2024-09-13T00:37:11.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:11 smithi088 bash[24895]: cephadm 2024-09-13T00:37:09.682888+0000 mgr.x (mgr.24490) 29 : cephadm [INF] Reconfiguring daemon node-exporter.a on smithi077 2024-09-13T00:37:11.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:11 smithi088 bash[24895]: cluster 2024-09-13T00:37:10.998324+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e191: 8 total, 8 up, 8 in 2024-09-13T00:37:11.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:11 smithi088 bash[24895]: cluster 2024-09-13T00:37:11.142800+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e192: 8 total, 8 up, 8 in 2024-09-13T00:37:11.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:11 smithi088 bash[24895]: audit 2024-09-13T00:37:11.395719+0000 mon.c (mon.2) 165 : audit [DBG] from='client.? 172.21.15.77:0/2393778847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:11.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:11 smithi088 bash[24895]: audit 2024-09-13T00:37:11.557928+0000 mon.a (mon.0) 1061 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:11.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:11 smithi088 bash[24895]: audit 2024-09-13T00:37:11.568290+0000 mon.a (mon.0) 1062 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:12.517 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:37:12.545 INFO:tasks.workunit.client.0.smithi077.stdout:write: Y 41943040~5 2024-09-13T00:37:12.786 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:37:12.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:12 smithi088 bash[24895]: cluster 2024-09-13T00:37:10.799746+0000 mgr.x (mgr.24490) 30 : cluster [DBG] pgmap v30: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 5.4 MiB/s wr, 378 op/s 2024-09-13T00:37:12.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:12 smithi088 bash[24895]: cephadm 2024-09-13T00:37:11.572208+0000 mgr.x (mgr.24490) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T00:37:12.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:12 smithi088 bash[24895]: cephadm 2024-09-13T00:37:11.600355+0000 mgr.x (mgr.24490) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi077 2024-09-13T00:37:12.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:12 smithi088 bash[24895]: cluster 2024-09-13T00:37:12.154184+0000 mon.a (mon.0) 1063 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2024-09-13T00:37:13.000 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:37:13.025 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:12 smithi077 bash[32040]: cluster 2024-09-13T00:37:10.799746+0000 mgr.x (mgr.24490) 30 : cluster [DBG] pgmap v30: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 5.4 MiB/s wr, 378 op/s 2024-09-13T00:37:13.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:12 smithi077 bash[32040]: cephadm 2024-09-13T00:37:11.572208+0000 mgr.x (mgr.24490) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T00:37:13.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:12 smithi077 bash[32040]: cephadm 2024-09-13T00:37:11.600355+0000 mgr.x (mgr.24490) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi077 2024-09-13T00:37:13.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:12 smithi077 bash[32040]: cluster 2024-09-13T00:37:12.154184+0000 mon.a (mon.0) 1063 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2024-09-13T00:37:13.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:12 smithi077 bash[22596]: cluster 2024-09-13T00:37:10.799746+0000 mgr.x (mgr.24490) 30 : cluster [DBG] pgmap v30: 137 pgs: 137 active+clean; 637 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 5.4 MiB/s wr, 378 op/s 2024-09-13T00:37:13.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:12 smithi077 bash[22596]: cephadm 2024-09-13T00:37:11.572208+0000 mgr.x (mgr.24490) 31 : cephadm [INF] Reconfiguring alertmanager.a (dependencies changed)... 2024-09-13T00:37:13.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:12 smithi077 bash[22596]: cephadm 2024-09-13T00:37:11.600355+0000 mgr.x (mgr.24490) 32 : cephadm [INF] Reconfiguring daemon alertmanager.a on smithi077 2024-09-13T00:37:13.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:12 smithi077 bash[22596]: cluster 2024-09-13T00:37:12.154184+0000 mon.a (mon.0) 1063 : cluster [DBG] osdmap e193: 8 total, 8 up, 8 in 2024-09-13T00:37:13.735 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare2 (4831 ms) 2024-09-13T00:37:13.735 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.SnapsSnapAfterPrepare 2024-09-13T00:37:13.793 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:37:13.815 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:37:13.888 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:37:14.479 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.285Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T00:37:14.480 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.285Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T00:37:14.480 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.285Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T00:37:14.480 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.285Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T00:37:14.480 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T00:37:14.480 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T00:37:14.480 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T00:37:14.480 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T00:37:14.480 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T00:37:14.480 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T00:37:14.480 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T00:37:14.481 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T00:37:14.481 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T00:37:14.481 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T00:37:14.481 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T00:37:14.481 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T00:37:14.481 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T00:37:14.481 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T00:37:14.481 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T00:37:14.481 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T00:37:14.481 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T00:37:14.481 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T00:37:14.482 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T00:37:14.482 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.286Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T00:37:14.482 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T00:37:14.482 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T00:37:14.482 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T00:37:14.482 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T00:37:14.482 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T00:37:14.482 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T00:37:14.482 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T00:37:14.482 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T00:37:14.482 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T00:37:14.482 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T00:37:14.483 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T00:37:14.483 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T00:37:14.483 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T00:37:14.483 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T00:37:14.483 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T00:37:14.483 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T00:37:14.483 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T00:37:14.483 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T00:37:14.483 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T00:37:14.483 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T00:37:14.484 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T00:37:14.484 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T00:37:14.484 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T00:37:14.484 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T00:37:14.484 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T00:37:14.484 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T00:37:14.485 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T00:37:14.485 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T00:37:14.485 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[72146]: ts=2024-09-13T00:37:14.287Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T00:37:14.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[32040]: cluster 2024-09-13T00:37:12.801619+0000 mgr.x (mgr.24490) 33 : cluster [DBG] pgmap v34: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 14 KiB/s wr, 154 op/s 2024-09-13T00:37:14.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[32040]: cluster 2024-09-13T00:37:13.591707+0000 mon.a (mon.0) 1064 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2024-09-13T00:37:14.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[32040]: audit 2024-09-13T00:37:13.766999+0000 mon.c (mon.2) 166 : audit [DBG] from='client.? 172.21.15.77:0/3699820727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:14.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[22596]: cluster 2024-09-13T00:37:12.801619+0000 mgr.x (mgr.24490) 33 : cluster [DBG] pgmap v34: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 14 KiB/s wr, 154 op/s 2024-09-13T00:37:14.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[22596]: cluster 2024-09-13T00:37:13.591707+0000 mon.a (mon.0) 1064 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2024-09-13T00:37:14.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:14 smithi077 bash[22596]: audit 2024-09-13T00:37:13.766999+0000 mon.c (mon.2) 166 : audit [DBG] from='client.? 172.21.15.77:0/3699820727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:14.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:14 smithi088 bash[24895]: cluster 2024-09-13T00:37:12.801619+0000 mgr.x (mgr.24490) 33 : cluster [DBG] pgmap v34: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 32 MiB/s rd, 14 KiB/s wr, 154 op/s 2024-09-13T00:37:14.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:14 smithi088 bash[24895]: cluster 2024-09-13T00:37:13.591707+0000 mon.a (mon.0) 1064 : cluster [DBG] osdmap e194: 8 total, 8 up, 8 in 2024-09-13T00:37:14.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:14 smithi088 bash[24895]: audit 2024-09-13T00:37:13.766999+0000 mon.c (mon.2) 166 : audit [DBG] from='client.? 172.21.15.77:0/3699820727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:15.637 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 8388609~8388609 2024-09-13T00:37:15.675 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 0~1000 2024-09-13T00:37:15.675 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 1010~1000 2024-09-13T00:37:15.675 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap2 2024-09-13T00:37:16.019 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:15 smithi077 bash[32040]: cluster 2024-09-13T00:37:14.597478+0000 mon.a (mon.0) 1065 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2024-09-13T00:37:16.019 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:15 smithi077 bash[22596]: cluster 2024-09-13T00:37:14.597478+0000 mon.a (mon.0) 1065 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2024-09-13T00:37:16.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:15 smithi088 bash[24895]: cluster 2024-09-13T00:37:14.597478+0000 mon.a (mon.0) 1065 : cluster [DBG] osdmap e195: 8 total, 8 up, 8 in 2024-09-13T00:37:16.884 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:16 smithi077 bash[32040]: cluster 2024-09-13T00:37:14.802427+0000 mgr.x (mgr.24490) 34 : cluster [DBG] pgmap v37: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 15 KiB/s wr, 162 op/s 2024-09-13T00:37:16.885 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:16 smithi077 bash[32040]: cluster 2024-09-13T00:37:15.613804+0000 mon.a (mon.0) 1066 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2024-09-13T00:37:16.885 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:16 smithi077 bash[32040]: audit 2024-09-13T00:37:16.122631+0000 mon.a (mon.0) 1067 : audit [DBG] from='client.? 172.21.15.77:0/2698565670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:16.885 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:16 smithi077 bash[32040]: cluster 2024-09-13T00:37:16.144378+0000 mon.a (mon.0) 1068 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2024-09-13T00:37:16.886 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:16 smithi077 bash[22596]: cluster 2024-09-13T00:37:14.802427+0000 mgr.x (mgr.24490) 34 : cluster [DBG] pgmap v37: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 15 KiB/s wr, 162 op/s 2024-09-13T00:37:16.886 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:16 smithi077 bash[22596]: cluster 2024-09-13T00:37:15.613804+0000 mon.a (mon.0) 1066 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2024-09-13T00:37:16.886 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:16 smithi077 bash[22596]: audit 2024-09-13T00:37:16.122631+0000 mon.a (mon.0) 1067 : audit [DBG] from='client.? 172.21.15.77:0/2698565670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:16.886 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:16 smithi077 bash[22596]: cluster 2024-09-13T00:37:16.144378+0000 mon.a (mon.0) 1068 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2024-09-13T00:37:17.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:16 smithi088 bash[24895]: cluster 2024-09-13T00:37:14.802427+0000 mgr.x (mgr.24490) 34 : cluster [DBG] pgmap v37: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 15 KiB/s wr, 162 op/s 2024-09-13T00:37:17.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:16 smithi088 bash[24895]: cluster 2024-09-13T00:37:15.613804+0000 mon.a (mon.0) 1066 : cluster [DBG] osdmap e196: 8 total, 8 up, 8 in 2024-09-13T00:37:17.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:16 smithi088 bash[24895]: audit 2024-09-13T00:37:16.122631+0000 mon.a (mon.0) 1067 : audit [DBG] from='client.? 172.21.15.77:0/2698565670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:17.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:16 smithi088 bash[24895]: cluster 2024-09-13T00:37:16.144378+0000 mon.a (mon.0) 1068 : cluster [DBG] osdmap e197: 8 total, 8 up, 8 in 2024-09-13T00:37:17.174 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 8388609~1000 2024-09-13T00:37:17.189 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 16777228~1000 2024-09-13T00:37:17.190 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 167772160 2024-09-13T00:37:17.210 INFO:tasks.workunit.client.0.smithi077.stdout:write: 2 83886079~8388609 2024-09-13T00:37:17.256 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap3 2024-09-13T00:37:18.026 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:17 smithi077 systemd[1]: Stopping Ceph alertmanager.a for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:37:18.026 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:17 smithi077 bash[58266]: ts=2024-09-13T00:37:17.868Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T00:37:18.458 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:18 smithi077 bash[22596]: cluster 2024-09-13T00:37:16.803307+0000 mgr.x (mgr.24490) 35 : cluster [DBG] pgmap v40: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:18.458 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:18 smithi077 bash[22596]: cluster 2024-09-13T00:37:17.149788+0000 mon.a (mon.0) 1069 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2024-09-13T00:37:18.458 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:18 smithi077 bash[32040]: cluster 2024-09-13T00:37:16.803307+0000 mgr.x (mgr.24490) 35 : cluster [DBG] pgmap v40: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:18.459 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:18 smithi077 bash[32040]: cluster 2024-09-13T00:37:17.149788+0000 mon.a (mon.0) 1069 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2024-09-13T00:37:18.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:18 smithi088 bash[24895]: cluster 2024-09-13T00:37:16.803307+0000 mgr.x (mgr.24490) 35 : cluster [DBG] pgmap v40: 137 pgs: 137 active+clean; 617 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:18.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:18 smithi088 bash[24895]: cluster 2024-09-13T00:37:17.149788+0000 mon.a (mon.0) 1069 : cluster [DBG] osdmap e198: 8 total, 8 up, 8 in 2024-09-13T00:37:18.776 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:18 smithi077 bash[72428]: ceph-9218306c-7165-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T00:37:18.776 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:18 smithi077 bash[72513]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T00:37:18.776 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:18 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T00:37:18.776 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:18 smithi077 systemd[1]: Stopped Ceph alertmanager.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:37:18.776 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:18 smithi077 systemd[1]: Started Ceph alertmanager.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:37:19.194 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:37:19.237 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 83886079~1 2024-09-13T00:37:19.252 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:37:19.457 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:19 smithi088 bash[24895]: cluster 2024-09-13T00:37:18.155521+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2024-09-13T00:37:19.457 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:19 smithi088 bash[24895]: audit 2024-09-13T00:37:18.479558+0000 mon.a (mon.0) 1071 : audit [DBG] from='client.? 172.21.15.77:0/2545172523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:19.457 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:19 smithi088 bash[24895]: audit 2024-09-13T00:37:18.752206+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:19.457 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:19 smithi088 bash[24895]: audit 2024-09-13T00:37:18.763421+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:19.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:19 smithi077 bash[22596]: cluster 2024-09-13T00:37:18.155521+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2024-09-13T00:37:19.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:19 smithi077 bash[22596]: audit 2024-09-13T00:37:18.479558+0000 mon.a (mon.0) 1071 : audit [DBG] from='client.? 172.21.15.77:0/2545172523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:19.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:19 smithi077 bash[22596]: audit 2024-09-13T00:37:18.752206+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:19.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:19 smithi077 bash[22596]: audit 2024-09-13T00:37:18.763421+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:19.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:19 smithi077 bash[32040]: cluster 2024-09-13T00:37:18.155521+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e199: 8 total, 8 up, 8 in 2024-09-13T00:37:19.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:19 smithi077 bash[32040]: audit 2024-09-13T00:37:18.479558+0000 mon.a (mon.0) 1071 : audit [DBG] from='client.? 172.21.15.77:0/2545172523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:19.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:19 smithi077 bash[32040]: audit 2024-09-13T00:37:18.752206+0000 mon.a (mon.0) 1072 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:19.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:19 smithi077 bash[32040]: audit 2024-09-13T00:37:18.763421+0000 mon.a (mon.0) 1073 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:20.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:20 smithi088 bash[24895]: cephadm 2024-09-13T00:37:18.768378+0000 mgr.x (mgr.24490) 36 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T00:37:20.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:20 smithi088 bash[24895]: cluster 2024-09-13T00:37:18.804243+0000 mgr.x (mgr.24490) 37 : cluster [DBG] pgmap v43: 137 pgs: 137 active+clean; 649 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 283 KiB/s rd, 8.0 MiB/s wr, 367 op/s 2024-09-13T00:37:20.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:20 smithi088 bash[24895]: cephadm 2024-09-13T00:37:19.063637+0000 mgr.x (mgr.24490) 38 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi088 2024-09-13T00:37:20.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:20 smithi088 bash[24895]: cluster 2024-09-13T00:37:19.176837+0000 mon.a (mon.0) 1074 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2024-09-13T00:37:20.526 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:20 smithi077 bash[72541]: ts=2024-09-13T00:37:20.279Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-13T00:37:20.526 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:20 smithi077 bash[72541]: ts=2024-09-13T00:37:20.279Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-13T00:37:20.526 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:20 smithi077 bash[72541]: ts=2024-09-13T00:37:20.281Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.77 port=9094 2024-09-13T00:37:20.526 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:20 smithi077 bash[72541]: ts=2024-09-13T00:37:20.282Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-13T00:37:20.526 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:20 smithi077 bash[72541]: ts=2024-09-13T00:37:20.312Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T00:37:20.526 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:20 smithi077 bash[72541]: ts=2024-09-13T00:37:20.313Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-13T00:37:20.526 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:20 smithi077 bash[72541]: ts=2024-09-13T00:37:20.316Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-13T00:37:20.527 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:20 smithi077 bash[72541]: ts=2024-09-13T00:37:20.316Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-13T00:37:20.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:20 smithi077 bash[22596]: cephadm 2024-09-13T00:37:18.768378+0000 mgr.x (mgr.24490) 36 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T00:37:20.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:20 smithi077 bash[22596]: cluster 2024-09-13T00:37:18.804243+0000 mgr.x (mgr.24490) 37 : cluster [DBG] pgmap v43: 137 pgs: 137 active+clean; 649 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 283 KiB/s rd, 8.0 MiB/s wr, 367 op/s 2024-09-13T00:37:20.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:20 smithi077 bash[22596]: cephadm 2024-09-13T00:37:19.063637+0000 mgr.x (mgr.24490) 38 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi088 2024-09-13T00:37:20.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:20 smithi077 bash[22596]: cluster 2024-09-13T00:37:19.176837+0000 mon.a (mon.0) 1074 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2024-09-13T00:37:20.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:20 smithi077 bash[32040]: cephadm 2024-09-13T00:37:18.768378+0000 mgr.x (mgr.24490) 36 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T00:37:20.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:20 smithi077 bash[32040]: cluster 2024-09-13T00:37:18.804243+0000 mgr.x (mgr.24490) 37 : cluster [DBG] pgmap v43: 137 pgs: 137 active+clean; 649 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 283 KiB/s rd, 8.0 MiB/s wr, 367 op/s 2024-09-13T00:37:20.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:20 smithi077 bash[32040]: cephadm 2024-09-13T00:37:19.063637+0000 mgr.x (mgr.24490) 38 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi088 2024-09-13T00:37:20.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:20 smithi077 bash[32040]: cluster 2024-09-13T00:37:19.176837+0000 mon.a (mon.0) 1074 : cluster [DBG] osdmap e200: 8 total, 8 up, 8 in 2024-09-13T00:37:20.944 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:20 smithi088 bash[53580]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:37:20] "GET /metrics HTTP/1.1" 200 38825 "" "Prometheus/2.43.0" 2024-09-13T00:37:21.462 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:21 smithi077 bash[32040]: cluster 2024-09-13T00:37:20.185652+0000 mon.a (mon.0) 1075 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2024-09-13T00:37:21.462 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:21 smithi077 bash[32040]: audit 2024-09-13T00:37:20.835764+0000 mon.a (mon.0) 1076 : audit [DBG] from='client.? 172.21.15.77:0/3729377675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:21.462 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:21 smithi077 bash[32040]: cluster 2024-09-13T00:37:21.152433+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2024-09-13T00:37:21.462 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:21 smithi077 bash[22596]: cluster 2024-09-13T00:37:20.185652+0000 mon.a (mon.0) 1075 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2024-09-13T00:37:21.462 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:21 smithi077 bash[22596]: audit 2024-09-13T00:37:20.835764+0000 mon.a (mon.0) 1076 : audit [DBG] from='client.? 172.21.15.77:0/3729377675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:21.462 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:21 smithi077 bash[22596]: cluster 2024-09-13T00:37:21.152433+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2024-09-13T00:37:21.476 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:21 smithi088 bash[24895]: cluster 2024-09-13T00:37:20.185652+0000 mon.a (mon.0) 1075 : cluster [DBG] osdmap e201: 8 total, 8 up, 8 in 2024-09-13T00:37:21.476 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:21 smithi088 bash[24895]: audit 2024-09-13T00:37:20.835764+0000 mon.a (mon.0) 1076 : audit [DBG] from='client.? 172.21.15.77:0/3729377675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:21.477 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:21 smithi088 bash[24895]: cluster 2024-09-13T00:37:21.152433+0000 mon.a (mon.0) 1077 : cluster [DBG] osdmap e202: 8 total, 8 up, 8 in 2024-09-13T00:37:21.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:21 smithi077 bash[22845]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:37:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:37:22.486 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:22 smithi088 bash[24895]: cluster 2024-09-13T00:37:20.804807+0000 mgr.x (mgr.24490) 39 : cluster [DBG] pgmap v46: 137 pgs: 137 active+clean; 649 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 283 KiB/s rd, 8.0 MiB/s wr, 367 op/s 2024-09-13T00:37:22.486 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:22 smithi088 bash[24895]: audit 2024-09-13T00:37:21.354587+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:22.486 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:22 smithi088 bash[24895]: cluster 2024-09-13T00:37:22.168338+0000 mon.a (mon.0) 1078 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2024-09-13T00:37:22.526 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:22 smithi077 bash[72541]: ts=2024-09-13T00:37:22.282Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000241165s 2024-09-13T00:37:22.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:22 smithi077 bash[22596]: cluster 2024-09-13T00:37:20.804807+0000 mgr.x (mgr.24490) 39 : cluster [DBG] pgmap v46: 137 pgs: 137 active+clean; 649 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 283 KiB/s rd, 8.0 MiB/s wr, 367 op/s 2024-09-13T00:37:22.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:22 smithi077 bash[22596]: audit 2024-09-13T00:37:21.354587+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:22.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:22 smithi077 bash[22596]: cluster 2024-09-13T00:37:22.168338+0000 mon.a (mon.0) 1078 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2024-09-13T00:37:22.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:22 smithi077 bash[32040]: cluster 2024-09-13T00:37:20.804807+0000 mgr.x (mgr.24490) 39 : cluster [DBG] pgmap v46: 137 pgs: 137 active+clean; 649 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 283 KiB/s rd, 8.0 MiB/s wr, 367 op/s 2024-09-13T00:37:22.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:22 smithi077 bash[32040]: audit 2024-09-13T00:37:21.354587+0000 mon.b (mon.1) 68 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:22.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:22 smithi077 bash[32040]: cluster 2024-09-13T00:37:22.168338+0000 mon.a (mon.0) 1078 : cluster [DBG] osdmap e203: 8 total, 8 up, 8 in 2024-09-13T00:37:23.260 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: after_prepare_snap 2024-09-13T00:37:23.350 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 systemd[1]: Stopping Ceph prometheus.a for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:37:23.351 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 bash[51748]: ts=2024-09-13T00:37:23.207Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T00:37:23.351 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 bash[51748]: ts=2024-09-13T00:37:23.207Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T00:37:23.351 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 bash[51748]: ts=2024-09-13T00:37:23.207Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T00:37:23.351 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 bash[51748]: ts=2024-09-13T00:37:23.207Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T00:37:23.351 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 bash[51748]: ts=2024-09-13T00:37:23.207Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T00:37:23.351 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 bash[51748]: ts=2024-09-13T00:37:23.207Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T00:37:23.351 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 bash[51748]: ts=2024-09-13T00:37:23.207Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T00:37:23.351 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 bash[51748]: ts=2024-09-13T00:37:23.207Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T00:37:23.351 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 bash[51748]: ts=2024-09-13T00:37:23.208Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T00:37:23.351 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 bash[51748]: ts=2024-09-13T00:37:23.208Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T00:37:23.351 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 bash[51748]: ts=2024-09-13T00:37:23.208Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T00:37:23.351 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 bash[51748]: ts=2024-09-13T00:37:23.208Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T00:37:24.008 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 bash[59916]: ceph-9218306c-7165-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T00:37:24.009 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 bash[59977]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T00:37:24.009 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T00:37:24.009 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 systemd[1]: Stopped Ceph prometheus.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:37:24.009 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:23 smithi088 systemd[1]: Started Ceph prometheus.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:37:24.350 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:24 smithi088 systemd[1]: Stopping Ceph node-exporter.b for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:37:24.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:24 smithi088 bash[24895]: cluster 2024-09-13T00:37:22.806470+0000 mgr.x (mgr.24490) 40 : cluster [DBG] pgmap v49: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 117 KiB/s rd, 4.0 MiB/s wr, 167 op/s 2024-09-13T00:37:24.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:24 smithi088 bash[24895]: audit 2024-09-13T00:37:23.225044+0000 mon.a (mon.0) 1079 : audit [DBG] from='client.? 172.21.15.77:0/4133438339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:24.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:24 smithi088 bash[24895]: audit 2024-09-13T00:37:23.944994+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:24 smithi088 bash[24895]: audit 2024-09-13T00:37:23.955072+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:24.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:24 smithi077 bash[22596]: cluster 2024-09-13T00:37:22.806470+0000 mgr.x (mgr.24490) 40 : cluster [DBG] pgmap v49: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 117 KiB/s rd, 4.0 MiB/s wr, 167 op/s 2024-09-13T00:37:24.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:24 smithi077 bash[22596]: audit 2024-09-13T00:37:23.225044+0000 mon.a (mon.0) 1079 : audit [DBG] from='client.? 172.21.15.77:0/4133438339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:24.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:24 smithi077 bash[22596]: audit 2024-09-13T00:37:23.944994+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:24.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:24 smithi077 bash[22596]: audit 2024-09-13T00:37:23.955072+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:24.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:24 smithi077 bash[32040]: cluster 2024-09-13T00:37:22.806470+0000 mgr.x (mgr.24490) 40 : cluster [DBG] pgmap v49: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 117 KiB/s rd, 4.0 MiB/s wr, 167 op/s 2024-09-13T00:37:24.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:24 smithi077 bash[32040]: audit 2024-09-13T00:37:23.225044+0000 mon.a (mon.0) 1079 : audit [DBG] from='client.? 172.21.15.77:0/4133438339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:24.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:24 smithi077 bash[32040]: audit 2024-09-13T00:37:23.944994+0000 mon.a (mon.0) 1080 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:24.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:24 smithi077 bash[32040]: audit 2024-09-13T00:37:23.955072+0000 mon.a (mon.0) 1081 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:25.251 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 41943040 2024-09-13T00:37:25.305 INFO:tasks.workunit.client.0.smithi077.stdout:write: * 0~1000 2024-09-13T00:37:25.314 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:37:25.473 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:37:25.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:25 smithi077 bash[32040]: cephadm 2024-09-13T00:37:23.955870+0000 mgr.x (mgr.24490) 41 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T00:37:25.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:25 smithi077 bash[32040]: cephadm 2024-09-13T00:37:23.956205+0000 mgr.x (mgr.24490) 42 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi088 2024-09-13T00:37:25.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:25 smithi077 bash[32040]: cluster 2024-09-13T00:37:24.210988+0000 mon.a (mon.0) 1082 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2024-09-13T00:37:25.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:25 smithi077 bash[22596]: cephadm 2024-09-13T00:37:23.955870+0000 mgr.x (mgr.24490) 41 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T00:37:25.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:25 smithi077 bash[22596]: cephadm 2024-09-13T00:37:23.956205+0000 mgr.x (mgr.24490) 42 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi088 2024-09-13T00:37:25.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:25 smithi077 bash[22596]: cluster 2024-09-13T00:37:24.210988+0000 mon.a (mon.0) 1082 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2024-09-13T00:37:25.559 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60051]: ceph-9218306c-7165-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T00:37:25.559 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:25 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T00:37:25.559 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60112]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T00:37:25.559 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:25 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T00:37:25.559 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:25 smithi088 systemd[1]: Stopped Ceph node-exporter.b for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:37:25.559 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:25 smithi088 systemd[1]: Started Ceph node-exporter.b for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:37:25.559 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[24895]: cephadm 2024-09-13T00:37:23.955870+0000 mgr.x (mgr.24490) 41 : cephadm [INF] Reconfiguring node-exporter.b (dependencies changed)... 2024-09-13T00:37:25.559 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[24895]: cephadm 2024-09-13T00:37:23.956205+0000 mgr.x (mgr.24490) 42 : cephadm [INF] Reconfiguring daemon node-exporter.b on smithi088 2024-09-13T00:37:25.559 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[24895]: cluster 2024-09-13T00:37:24.210988+0000 mon.a (mon.0) 1082 : cluster [DBG] osdmap e204: 8 total, 8 up, 8 in 2024-09-13T00:37:25.853 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.672Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T00:37:25.853 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.672Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T00:37:25.853 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.672Z caller=main.go:570 level=info host_details="(Linux 5.15.0-121-generic #131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024 x86_64 smithi088 (none))" 2024-09-13T00:37:25.853 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.672Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T00:37:25.853 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.672Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T00:37:25.854 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.675Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T00:37:26.227 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.675Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T00:37:26.227 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.676Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T00:37:26.227 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.676Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T00:37:26.227 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.678Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T00:37:26.227 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.678Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.529µs 2024-09-13T00:37:26.227 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.678Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T00:37:26.227 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.681Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-13T00:37:26.227 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.706Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-13T00:37:26.227 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.706Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-13T00:37:26.228 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.706Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=24.893µs wal_replay_duration=28.169024ms wbl_replay_duration=119ns total_replay_duration=28.217357ms 2024-09-13T00:37:26.228 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.709Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T00:37:26.228 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.709Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T00:37:26.228 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.709Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T00:37:26.228 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.735Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=26.078352ms db_storage=1.076µs remote_storage=1.31µs web_handler=475ns query_engine=832ns scrape=298.55µs scrape_sd=137.547µs notify=14.133µs notify_sd=16.765µs rules=25.188194ms tracing=5.92µs 2024-09-13T00:37:26.228 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.735Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T00:37:26.228 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:25 smithi088 bash[60003]: ts=2024-09-13T00:37:25.735Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T00:37:26.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:26 smithi077 bash[22596]: cluster 2024-09-13T00:37:24.807301+0000 mgr.x (mgr.24490) 43 : cluster [DBG] pgmap v51: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 102 KiB/s rd, 3.5 MiB/s wr, 145 op/s 2024-09-13T00:37:26.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:26 smithi077 bash[22596]: cluster 2024-09-13T00:37:25.236193+0000 mon.a (mon.0) 1083 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2024-09-13T00:37:26.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:26 smithi077 bash[22596]: audit 2024-09-13T00:37:25.480352+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:26.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:26 smithi077 bash[22596]: audit 2024-09-13T00:37:25.493363+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:26.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:26 smithi077 bash[22596]: cephadm 2024-09-13T00:37:25.493882+0000 mgr.x (mgr.24490) 44 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-09-13T00:37:26.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:26 smithi077 bash[22596]: cephadm 2024-09-13T00:37:25.546217+0000 mgr.x (mgr.24490) 45 : cephadm [INF] Reconfiguring daemon grafana.a on smithi088 2024-09-13T00:37:26.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:26 smithi077 bash[22596]: audit 2024-09-13T00:37:25.636319+0000 mon.c (mon.2) 167 : audit [DBG] from='client.? 172.21.15.77:0/524975694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:26.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:26 smithi077 bash[32040]: cluster 2024-09-13T00:37:24.807301+0000 mgr.x (mgr.24490) 43 : cluster [DBG] pgmap v51: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 102 KiB/s rd, 3.5 MiB/s wr, 145 op/s 2024-09-13T00:37:26.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:26 smithi077 bash[32040]: cluster 2024-09-13T00:37:25.236193+0000 mon.a (mon.0) 1083 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2024-09-13T00:37:26.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:26 smithi077 bash[32040]: audit 2024-09-13T00:37:25.480352+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:26.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:26 smithi077 bash[32040]: audit 2024-09-13T00:37:25.493363+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:26.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:26 smithi077 bash[32040]: cephadm 2024-09-13T00:37:25.493882+0000 mgr.x (mgr.24490) 44 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-09-13T00:37:26.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:26 smithi077 bash[32040]: cephadm 2024-09-13T00:37:25.546217+0000 mgr.x (mgr.24490) 45 : cephadm [INF] Reconfiguring daemon grafana.a on smithi088 2024-09-13T00:37:26.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:26 smithi077 bash[32040]: audit 2024-09-13T00:37:25.636319+0000 mon.c (mon.2) 167 : audit [DBG] from='client.? 172.21.15.77:0/524975694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:26.575 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:26 smithi088 bash[24895]: cluster 2024-09-13T00:37:24.807301+0000 mgr.x (mgr.24490) 43 : cluster [DBG] pgmap v51: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 102 KiB/s rd, 3.5 MiB/s wr, 145 op/s 2024-09-13T00:37:26.575 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:26 smithi088 bash[24895]: cluster 2024-09-13T00:37:25.236193+0000 mon.a (mon.0) 1083 : cluster [DBG] osdmap e205: 8 total, 8 up, 8 in 2024-09-13T00:37:26.575 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:26 smithi088 bash[24895]: audit 2024-09-13T00:37:25.480352+0000 mon.a (mon.0) 1084 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:26.575 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:26 smithi088 bash[24895]: audit 2024-09-13T00:37:25.493363+0000 mon.a (mon.0) 1085 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:26.575 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:26 smithi088 bash[24895]: cephadm 2024-09-13T00:37:25.493882+0000 mgr.x (mgr.24490) 44 : cephadm [INF] Reconfiguring grafana.a (dependencies changed)... 2024-09-13T00:37:26.575 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:26 smithi088 bash[24895]: cephadm 2024-09-13T00:37:25.546217+0000 mgr.x (mgr.24490) 45 : cephadm [INF] Reconfiguring daemon grafana.a on smithi088 2024-09-13T00:37:26.575 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:26 smithi088 bash[24895]: audit 2024-09-13T00:37:25.636319+0000 mon.c (mon.2) 167 : audit [DBG] from='client.? 172.21.15.77:0/524975694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:27.753 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[24895]: cluster 2024-09-13T00:37:26.499414+0000 mon.a (mon.0) 1086 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2024-09-13T00:37:27.754 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.642Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-13T00:37:27.754 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.642Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-13T00:37:27.754 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.642Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-13T00:37:27.754 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.642Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-13T00:37:27.754 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.643Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-13T00:37:27.754 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.643Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-13T00:37:27.754 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.643Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-13T00:37:27.754 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.643Z caller=node_exporter.go:117 level=info collector=arp 2024-09-13T00:37:27.754 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.643Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-13T00:37:27.754 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.643Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-13T00:37:27.754 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.643Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-13T00:37:27.755 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.643Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-13T00:37:27.755 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-13T00:37:27.755 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-13T00:37:27.755 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-13T00:37:27.755 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-13T00:37:27.755 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=edac 2024-09-13T00:37:27.755 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-13T00:37:27.755 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-13T00:37:27.755 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-13T00:37:27.755 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-13T00:37:27.756 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-13T00:37:27.756 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-13T00:37:27.756 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-13T00:37:27.756 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-13T00:37:27.756 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-13T00:37:27.756 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-13T00:37:27.756 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-13T00:37:27.756 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-13T00:37:27.756 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-13T00:37:27.756 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-13T00:37:27.756 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-13T00:37:27.757 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-13T00:37:27.757 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=os 2024-09-13T00:37:27.757 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-13T00:37:27.757 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-13T00:37:27.757 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-13T00:37:27.757 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-13T00:37:27.757 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-13T00:37:27.757 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-13T00:37:27.757 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-13T00:37:27.757 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=stat 2024-09-13T00:37:27.757 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-13T00:37:27.757 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-13T00:37:27.758 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-13T00:37:27.758 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=time 2024-09-13T00:37:27.758 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-13T00:37:27.758 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=uname 2024-09-13T00:37:27.758 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-13T00:37:27.758 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-13T00:37:27.758 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-13T00:37:27.758 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-13T00:37:27.758 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:37:27 smithi088 bash[60140]: ts=2024-09-13T00:37:27.644Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-13T00:37:27.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:27 smithi077 bash[32040]: cluster 2024-09-13T00:37:26.499414+0000 mon.a (mon.0) 1086 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2024-09-13T00:37:27.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:27 smithi077 bash[22596]: cluster 2024-09-13T00:37:26.499414+0000 mon.a (mon.0) 1086 : cluster [DBG] osdmap e206: 8 total, 8 up, 8 in 2024-09-13T00:37:28.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:28 smithi077 bash[22596]: cluster 2024-09-13T00:37:26.808077+0000 mgr.x (mgr.24490) 46 : cluster [DBG] pgmap v54: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 101 KiB/s rd, 3.5 MiB/s wr, 144 op/s 2024-09-13T00:37:28.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:28 smithi077 bash[22596]: cluster 2024-09-13T00:37:27.508682+0000 mon.a (mon.0) 1087 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2024-09-13T00:37:28.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:28 smithi077 bash[22596]: audit 2024-09-13T00:37:28.003629+0000 mon.c (mon.2) 168 : audit [DBG] from='client.? 172.21.15.77:0/2309712204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:28.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:28 smithi077 bash[32040]: cluster 2024-09-13T00:37:26.808077+0000 mgr.x (mgr.24490) 46 : cluster [DBG] pgmap v54: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 101 KiB/s rd, 3.5 MiB/s wr, 144 op/s 2024-09-13T00:37:28.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:28 smithi077 bash[32040]: cluster 2024-09-13T00:37:27.508682+0000 mon.a (mon.0) 1087 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2024-09-13T00:37:28.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:28 smithi077 bash[32040]: audit 2024-09-13T00:37:28.003629+0000 mon.c (mon.2) 168 : audit [DBG] from='client.? 172.21.15.77:0/2309712204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:28.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:28 smithi088 bash[24895]: cluster 2024-09-13T00:37:26.808077+0000 mgr.x (mgr.24490) 46 : cluster [DBG] pgmap v54: 137 pgs: 137 active+clean; 673 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 101 KiB/s rd, 3.5 MiB/s wr, 144 op/s 2024-09-13T00:37:28.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:28 smithi088 bash[24895]: cluster 2024-09-13T00:37:27.508682+0000 mon.a (mon.0) 1087 : cluster [DBG] osdmap e207: 8 total, 8 up, 8 in 2024-09-13T00:37:28.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:28 smithi088 bash[24895]: audit 2024-09-13T00:37:28.003629+0000 mon.c (mon.2) 168 : audit [DBG] from='client.? 172.21.15.77:0/2309712204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:29.150 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.SnapsSnapAfterPrepare (15415 ms) 2024-09-13T00:37:29.150 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.SnapsSnapExpandAfterPrepare 2024-09-13T00:37:29.232 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:37:29.261 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:37:29.370 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:37:29.732 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:29 smithi088 bash[24895]: cluster 2024-09-13T00:37:28.518149+0000 mon.a (mon.0) 1088 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2024-09-13T00:37:29.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:29 smithi077 bash[22596]: cluster 2024-09-13T00:37:28.518149+0000 mon.a (mon.0) 1088 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2024-09-13T00:37:29.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:29 smithi077 bash[32040]: cluster 2024-09-13T00:37:28.518149+0000 mon.a (mon.0) 1088 : cluster [DBG] osdmap e208: 8 total, 8 up, 8 in 2024-09-13T00:37:30.466 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:30 smithi088 systemd[1]: Stopping Ceph grafana.a for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:37:30.570 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 8388609~8388609 2024-09-13T00:37:30.599 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 0~1000 2024-09-13T00:37:30.599 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 1010~1000 2024-09-13T00:37:30.599 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap2 2024-09-13T00:37:30.776 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:30 smithi077 bash[72541]: ts=2024-09-13T00:37:30.284Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002200712s 2024-09-13T00:37:30.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:30 smithi077 bash[22596]: cluster 2024-09-13T00:37:28.808956+0000 mgr.x (mgr.24490) 47 : cluster [DBG] pgmap v57: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 536 op/s 2024-09-13T00:37:30.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:30 smithi077 bash[22596]: cluster 2024-09-13T00:37:29.529731+0000 mon.a (mon.0) 1089 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2024-09-13T00:37:30.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:30 smithi077 bash[22596]: audit 2024-09-13T00:37:30.367162+0000 mon.c (mon.2) 169 : audit [DBG] from='client.? 172.21.15.77:0/701824710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:30.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:30 smithi077 bash[32040]: cluster 2024-09-13T00:37:28.808956+0000 mgr.x (mgr.24490) 47 : cluster [DBG] pgmap v57: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 536 op/s 2024-09-13T00:37:30.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:30 smithi077 bash[32040]: cluster 2024-09-13T00:37:29.529731+0000 mon.a (mon.0) 1089 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2024-09-13T00:37:30.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:30 smithi077 bash[32040]: audit 2024-09-13T00:37:30.367162+0000 mon.c (mon.2) 169 : audit [DBG] from='client.? 172.21.15.77:0/701824710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:30.850 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:30 smithi088 bash[60429]: ceph-9218306c-7165-11ef-bceb-c7b262605968-grafana-a 2024-09-13T00:37:30.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:30 smithi088 bash[24895]: cluster 2024-09-13T00:37:28.808956+0000 mgr.x (mgr.24490) 47 : cluster [DBG] pgmap v57: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 536 op/s 2024-09-13T00:37:30.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:30 smithi088 bash[24895]: cluster 2024-09-13T00:37:29.529731+0000 mon.a (mon.0) 1089 : cluster [DBG] osdmap e209: 8 total, 8 up, 8 in 2024-09-13T00:37:30.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:30 smithi088 bash[24895]: audit 2024-09-13T00:37:30.367162+0000 mon.c (mon.2) 169 : audit [DBG] from='client.? 172.21.15.77:0/701824710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:31.350 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:30 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T00:37:31.350 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:30 smithi088 bash[60491]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-grafana-a 2024-09-13T00:37:31.350 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:30 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-13T00:37:31.351 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:30 smithi088 systemd[1]: Stopped Ceph grafana.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:37:31.351 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:30 smithi088 systemd[1]: Started Ceph grafana.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:37:31.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:31 smithi077 bash[32040]: cluster 2024-09-13T00:37:30.543232+0000 mon.a (mon.0) 1090 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2024-09-13T00:37:31.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:31 smithi077 bash[32040]: audit 2024-09-13T00:37:30.933337+0000 mon.a (mon.0) 1091 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:31.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:31 smithi077 bash[32040]: audit 2024-09-13T00:37:30.943598+0000 mon.a (mon.0) 1092 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:31.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:31 smithi077 bash[32040]: audit 2024-09-13T00:37:30.947478+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T00:37:31.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:31 smithi077 bash[32040]: audit 2024-09-13T00:37:30.951192+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:37:31.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:31 smithi077 bash[32040]: audit 2024-09-13T00:37:30.954326+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T00:37:31.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:31 smithi077 bash[32040]: audit 2024-09-13T00:37:30.995263+0000 mon.b (mon.1) 72 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:37:31.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:31 smithi077 bash[32040]: cluster 2024-09-13T00:37:31.157285+0000 mon.a (mon.0) 1093 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2024-09-13T00:37:31.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:31 smithi077 bash[22596]: cluster 2024-09-13T00:37:30.543232+0000 mon.a (mon.0) 1090 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2024-09-13T00:37:31.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:31 smithi077 bash[22596]: audit 2024-09-13T00:37:30.933337+0000 mon.a (mon.0) 1091 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:31.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:31 smithi077 bash[22596]: audit 2024-09-13T00:37:30.943598+0000 mon.a (mon.0) 1092 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:31.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:31 smithi077 bash[22596]: audit 2024-09-13T00:37:30.947478+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T00:37:31.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:31 smithi077 bash[22596]: audit 2024-09-13T00:37:30.951192+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:37:31.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:31 smithi077 bash[22596]: audit 2024-09-13T00:37:30.954326+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T00:37:31.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:31 smithi077 bash[22596]: audit 2024-09-13T00:37:30.995263+0000 mon.b (mon.1) 72 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:37:31.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:31 smithi077 bash[22596]: cluster 2024-09-13T00:37:31.157285+0000 mon.a (mon.0) 1093 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2024-09-13T00:37:31.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:31 smithi088 bash[24895]: cluster 2024-09-13T00:37:30.543232+0000 mon.a (mon.0) 1090 : cluster [DBG] osdmap e210: 8 total, 8 up, 8 in 2024-09-13T00:37:31.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:31 smithi088 bash[24895]: audit 2024-09-13T00:37:30.933337+0000 mon.a (mon.0) 1091 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:31.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:31 smithi088 bash[24895]: audit 2024-09-13T00:37:30.943598+0000 mon.a (mon.0) 1092 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:31.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:31 smithi088 bash[24895]: audit 2024-09-13T00:37:30.947478+0000 mon.b (mon.1) 69 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T00:37:31.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:31 smithi088 bash[24895]: audit 2024-09-13T00:37:30.951192+0000 mon.b (mon.1) 70 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:37:31.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:31 smithi088 bash[24895]: audit 2024-09-13T00:37:30.954326+0000 mon.b (mon.1) 71 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T00:37:31.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:31 smithi088 bash[24895]: audit 2024-09-13T00:37:30.995263+0000 mon.b (mon.1) 72 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:37:31.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:31 smithi088 bash[24895]: cluster 2024-09-13T00:37:31.157285+0000 mon.a (mon.0) 1093 : cluster [DBG] osdmap e211: 8 total, 8 up, 8 in 2024-09-13T00:37:32.177 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=settings t=2024-09-13T00:37:32.162124208Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-13T00:37:32.178 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=settings t=2024-09-13T00:37:32.162334621Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-13T00:37:32.178 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=settings t=2024-09-13T00:37:32.162361635Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-13T00:37:32.178 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=settings t=2024-09-13T00:37:32.162378155Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-13T00:37:32.178 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=settings t=2024-09-13T00:37:32.16239242Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-13T00:37:32.178 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=settings t=2024-09-13T00:37:32.162403836Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-13T00:37:32.178 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=settings t=2024-09-13T00:37:32.162418526Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-13T00:37:32.178 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=settings t=2024-09-13T00:37:32.162429055Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-13T00:37:32.178 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=settings t=2024-09-13T00:37:32.162444216Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-13T00:37:32.178 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=settings t=2024-09-13T00:37:32.16245567Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-13T00:37:32.178 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=settings t=2024-09-13T00:37:32.162466788Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-13T00:37:32.178 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=settings t=2024-09-13T00:37:32.162477481Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-13T00:37:32.178 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=settings t=2024-09-13T00:37:32.162489045Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-13T00:37:32.179 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=settings t=2024-09-13T00:37:32.162500802Z level=info msg="App mode production" 2024-09-13T00:37:32.179 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=sqlstore t=2024-09-13T00:37:32.162578922Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-13T00:37:32.179 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=sqlstore t=2024-09-13T00:37:32.16260901Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-13T00:37:32.179 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=migrator t=2024-09-13T00:37:32.181185792Z level=info msg="Starting DB migrations" 2024-09-13T00:37:32.179 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 8388609~1000 2024-09-13T00:37:32.189 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 16777228~1000 2024-09-13T00:37:32.189 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 167772160 2024-09-13T00:37:32.210 INFO:tasks.workunit.client.0.smithi077.stdout:write: 2 83886079~8388609 2024-09-13T00:37:32.258 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap3 2024-09-13T00:37:32.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=migrator t=2024-09-13T00:37:32.187323583Z level=info msg="migrations completed" performed=0 skipped=481 duration=564.202µs 2024-09-13T00:37:32.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=sqlstore t=2024-09-13T00:37:32.187827447Z level=info msg="Created default organization" 2024-09-13T00:37:32.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=plugin.loader t=2024-09-13T00:37:32.266145229Z level=info msg="Plugin registered" pluginID=input 2024-09-13T00:37:32.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=plugin.loader t=2024-09-13T00:37:32.299516597Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-13T00:37:32.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=plugin.loader t=2024-09-13T00:37:32.299548324Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-13T00:37:32.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=secrets t=2024-09-13T00:37:32.299594159Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-13T00:37:32.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=query_data t=2024-09-13T00:37:32.30164075Z level=info msg="Query Service initialization" 2024-09-13T00:37:32.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[60519]: logger=live.push_http t=2024-09-13T00:37:32.306172209Z level=info msg="Live Push Gateway initialization" 2024-09-13T00:37:32.810 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:32 smithi077 bash[32040]: cluster 2024-09-13T00:37:30.809834+0000 mgr.x (mgr.24490) 48 : cluster [DBG] pgmap v60: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 536 op/s 2024-09-13T00:37:32.810 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:32 smithi077 bash[32040]: audit 2024-09-13T00:37:30.947942+0000 mgr.x (mgr.24490) 49 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T00:37:32.810 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:32 smithi077 bash[32040]: audit 2024-09-13T00:37:30.951576+0000 mgr.x (mgr.24490) 50 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:37:32.810 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:32 smithi077 bash[32040]: audit 2024-09-13T00:37:30.954662+0000 mgr.x (mgr.24490) 51 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T00:37:32.810 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:32 smithi077 bash[32040]: cephadm 2024-09-13T00:37:30.998170+0000 mgr.x (mgr.24490) 52 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T00:37:32.810 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:32 smithi077 bash[32040]: cephadm 2024-09-13T00:37:30.998707+0000 mgr.x (mgr.24490) 53 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T00:37:32.810 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:32 smithi077 bash[32040]: cluster 2024-09-13T00:37:32.163360+0000 mon.a (mon.0) 1094 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2024-09-13T00:37:32.811 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:32 smithi077 bash[22596]: cluster 2024-09-13T00:37:30.809834+0000 mgr.x (mgr.24490) 48 : cluster [DBG] pgmap v60: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 536 op/s 2024-09-13T00:37:32.811 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:32 smithi077 bash[22596]: audit 2024-09-13T00:37:30.947942+0000 mgr.x (mgr.24490) 49 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T00:37:32.811 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:32 smithi077 bash[22596]: audit 2024-09-13T00:37:30.951576+0000 mgr.x (mgr.24490) 50 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:37:32.811 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:32 smithi077 bash[22596]: audit 2024-09-13T00:37:30.954662+0000 mgr.x (mgr.24490) 51 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T00:37:32.811 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:32 smithi077 bash[22596]: cephadm 2024-09-13T00:37:30.998170+0000 mgr.x (mgr.24490) 52 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T00:37:32.811 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:32 smithi077 bash[22596]: cephadm 2024-09-13T00:37:30.998707+0000 mgr.x (mgr.24490) 53 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T00:37:32.811 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:32 smithi077 bash[22596]: cluster 2024-09-13T00:37:32.163360+0000 mon.a (mon.0) 1094 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2024-09-13T00:37:32.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[24895]: cluster 2024-09-13T00:37:30.809834+0000 mgr.x (mgr.24490) 48 : cluster [DBG] pgmap v60: 137 pgs: 137 active+clean; 693 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 536 op/s 2024-09-13T00:37:32.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[24895]: audit 2024-09-13T00:37:30.947942+0000 mgr.x (mgr.24490) 49 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-13T00:37:32.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[24895]: audit 2024-09-13T00:37:30.951576+0000 mgr.x (mgr.24490) 50 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:37:32.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[24895]: audit 2024-09-13T00:37:30.954662+0000 mgr.x (mgr.24490) 51 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-13T00:37:32.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[24895]: cephadm 2024-09-13T00:37:30.998170+0000 mgr.x (mgr.24490) 52 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T00:37:32.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[24895]: cephadm 2024-09-13T00:37:30.998707+0000 mgr.x (mgr.24490) 53 : cephadm [INF] Upgrade: Need to upgrade myself (mgr.x) 2024-09-13T00:37:32.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:32 smithi088 bash[24895]: cluster 2024-09-13T00:37:32.163360+0000 mon.a (mon.0) 1094 : cluster [DBG] osdmap e212: 8 total, 8 up, 8 in 2024-09-13T00:37:33.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:33 smithi088 bash[24895]: audit 2024-09-13T00:37:32.733229+0000 mon.a (mon.0) 1095 : audit [DBG] from='client.? 172.21.15.77:0/3551770199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:33.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:33 smithi088 bash[24895]: cluster 2024-09-13T00:37:33.168401+0000 mon.a (mon.0) 1096 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2024-09-13T00:37:33.851 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:33 smithi088 bash[60519]: logger=infra.usagestats.collector t=2024-09-13T00:37:33.567029466Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-13T00:37:33.851 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:33 smithi088 bash[60519]: logger=provisioning.datasources t=2024-09-13T00:37:33.70501518Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2024-09-13T00:37:33.851 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:33 smithi088 bash[60519]: logger=provisioning.datasources t=2024-09-13T00:37:33.705084959Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-13T00:37:33.851 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:33 smithi088 bash[60519]: logger=provisioning.datasources t=2024-09-13T00:37:33.716588908Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-13T00:37:34.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:33 smithi077 bash[22596]: audit 2024-09-13T00:37:32.733229+0000 mon.a (mon.0) 1095 : audit [DBG] from='client.? 172.21.15.77:0/3551770199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:34.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:33 smithi077 bash[22596]: cluster 2024-09-13T00:37:33.168401+0000 mon.a (mon.0) 1096 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2024-09-13T00:37:34.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:33 smithi077 bash[32040]: audit 2024-09-13T00:37:32.733229+0000 mon.a (mon.0) 1095 : audit [DBG] from='client.? 172.21.15.77:0/3551770199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:34.026 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:33 smithi077 bash[32040]: cluster 2024-09-13T00:37:33.168401+0000 mon.a (mon.0) 1096 : cluster [DBG] osdmap e213: 8 total, 8 up, 8 in 2024-09-13T00:37:34.188 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:37:34.249 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 83886079~1 2024-09-13T00:37:34.273 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:37:34.350 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:33 smithi088 bash[60519]: logger=provisioning.alerting t=2024-09-13T00:37:33.946701196Z level=info msg="starting to provision alerting" 2024-09-13T00:37:34.350 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:33 smithi088 bash[60519]: logger=provisioning.alerting t=2024-09-13T00:37:33.946871603Z level=info msg="finished to provision alerting" 2024-09-13T00:37:34.350 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:33 smithi088 bash[60519]: logger=grafanaStorageLogger t=2024-09-13T00:37:33.952351907Z level=info msg="storage starting" 2024-09-13T00:37:34.350 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:33 smithi088 bash[60519]: logger=ngalert.state.manager t=2024-09-13T00:37:33.952448578Z level=info msg="Warming state cache for startup" 2024-09-13T00:37:34.351 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:33 smithi088 bash[60519]: logger=http.server t=2024-09-13T00:37:33.955018578Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-13T00:37:34.351 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:33 smithi088 bash[60519]: logger=ngalert.state.manager t=2024-09-13T00:37:33.98159098Z level=info msg="State cache has been initialized" states=0 duration=29.138361ms 2024-09-13T00:37:34.351 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:33 smithi088 bash[60519]: logger=ticker t=2024-09-13T00:37:33.981723885Z level=info msg=starting first_tick=2024-09-13T00:37:40Z 2024-09-13T00:37:34.351 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:37:33 smithi088 bash[60519]: logger=ngalert.multiorg.alertmanager t=2024-09-13T00:37:33.981747701Z level=info msg="starting MultiOrg Alertmanager" 2024-09-13T00:37:34.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:34 smithi088 bash[24895]: cluster 2024-09-13T00:37:32.811966+0000 mgr.x (mgr.24490) 54 : cluster [DBG] pgmap v63: 137 pgs: 137 active+clean; 705 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 141 op/s 2024-09-13T00:37:34.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:34 smithi088 bash[24895]: audit 2024-09-13T00:37:33.739641+0000 mon.a (mon.0) 1097 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:34.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:34 smithi088 bash[24895]: audit 2024-09-13T00:37:33.745401+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:37:34.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:34 smithi088 bash[24895]: audit 2024-09-13T00:37:33.747324+0000 mon.a (mon.0) 1098 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:37:34.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:34 smithi088 bash[24895]: audit 2024-09-13T00:37:33.748482+0000 mon.b (mon.1) 74 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:37:34.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:34 smithi088 bash[24895]: audit 2024-09-13T00:37:33.750516+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:37:34.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:34 smithi088 bash[24895]: cluster 2024-09-13T00:37:34.171482+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2024-09-13T00:37:35.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:34 smithi077 bash[22596]: cluster 2024-09-13T00:37:32.811966+0000 mgr.x (mgr.24490) 54 : cluster [DBG] pgmap v63: 137 pgs: 137 active+clean; 705 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 141 op/s 2024-09-13T00:37:35.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:34 smithi077 bash[22596]: audit 2024-09-13T00:37:33.739641+0000 mon.a (mon.0) 1097 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:35.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:34 smithi077 bash[22596]: audit 2024-09-13T00:37:33.745401+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:37:35.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:34 smithi077 bash[22596]: audit 2024-09-13T00:37:33.747324+0000 mon.a (mon.0) 1098 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:37:35.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:34 smithi077 bash[22596]: audit 2024-09-13T00:37:33.748482+0000 mon.b (mon.1) 74 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:37:35.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:34 smithi077 bash[22596]: audit 2024-09-13T00:37:33.750516+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:37:35.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:34 smithi077 bash[22596]: cluster 2024-09-13T00:37:34.171482+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2024-09-13T00:37:35.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:34 smithi077 bash[32040]: cluster 2024-09-13T00:37:32.811966+0000 mgr.x (mgr.24490) 54 : cluster [DBG] pgmap v63: 137 pgs: 137 active+clean; 705 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 141 op/s 2024-09-13T00:37:35.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:34 smithi077 bash[32040]: audit 2024-09-13T00:37:33.739641+0000 mon.a (mon.0) 1097 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:35.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:34 smithi077 bash[32040]: audit 2024-09-13T00:37:33.745401+0000 mon.b (mon.1) 73 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:37:35.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:34 smithi077 bash[32040]: audit 2024-09-13T00:37:33.747324+0000 mon.a (mon.0) 1098 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:37:35.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:34 smithi077 bash[32040]: audit 2024-09-13T00:37:33.748482+0000 mon.b (mon.1) 74 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:37:35.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:34 smithi077 bash[32040]: audit 2024-09-13T00:37:33.750516+0000 mon.b (mon.1) 75 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:37:35.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:34 smithi077 bash[32040]: cluster 2024-09-13T00:37:34.171482+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e214: 8 total, 8 up, 8 in 2024-09-13T00:37:35.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:35 smithi077 bash[22596]: cephadm 2024-09-13T00:37:33.732251+0000 mgr.x (mgr.24490) 55 : cephadm [INF] Upgrade: Updating mgr.y 2024-09-13T00:37:35.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:35 smithi077 bash[22596]: cephadm 2024-09-13T00:37:33.752501+0000 mgr.x (mgr.24490) 56 : cephadm [INF] Deploying daemon mgr.y on smithi077 2024-09-13T00:37:35.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:35 smithi077 bash[22596]: audit 2024-09-13T00:37:35.097593+0000 mon.c (mon.2) 170 : audit [DBG] from='client.? 172.21.15.77:0/3501780463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:35.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:35 smithi077 bash[22596]: cluster 2024-09-13T00:37:35.167587+0000 mon.a (mon.0) 1100 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2024-09-13T00:37:35.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:35 smithi077 bash[32040]: cephadm 2024-09-13T00:37:33.732251+0000 mgr.x (mgr.24490) 55 : cephadm [INF] Upgrade: Updating mgr.y 2024-09-13T00:37:35.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:35 smithi077 bash[32040]: cephadm 2024-09-13T00:37:33.752501+0000 mgr.x (mgr.24490) 56 : cephadm [INF] Deploying daemon mgr.y on smithi077 2024-09-13T00:37:35.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:35 smithi077 bash[32040]: audit 2024-09-13T00:37:35.097593+0000 mon.c (mon.2) 170 : audit [DBG] from='client.? 172.21.15.77:0/3501780463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:35.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:35 smithi077 bash[32040]: cluster 2024-09-13T00:37:35.167587+0000 mon.a (mon.0) 1100 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2024-09-13T00:37:35.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:35 smithi088 bash[24895]: cephadm 2024-09-13T00:37:33.732251+0000 mgr.x (mgr.24490) 55 : cephadm [INF] Upgrade: Updating mgr.y 2024-09-13T00:37:35.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:35 smithi088 bash[24895]: cephadm 2024-09-13T00:37:33.752501+0000 mgr.x (mgr.24490) 56 : cephadm [INF] Deploying daemon mgr.y on smithi077 2024-09-13T00:37:35.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:35 smithi088 bash[24895]: audit 2024-09-13T00:37:35.097593+0000 mon.c (mon.2) 170 : audit [DBG] from='client.? 172.21.15.77:0/3501780463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:35.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:35 smithi088 bash[24895]: cluster 2024-09-13T00:37:35.167587+0000 mon.a (mon.0) 1100 : cluster [DBG] osdmap e215: 8 total, 8 up, 8 in 2024-09-13T00:37:36.692 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:36 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:36.693 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:37:36 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:36.693 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:36 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:36.693 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:36 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:36.693 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:36 smithi077 bash[22596]: cluster 2024-09-13T00:37:34.812779+0000 mgr.x (mgr.24490) 57 : cluster [DBG] pgmap v66: 137 pgs: 137 active+clean; 705 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 141 op/s 2024-09-13T00:37:36.694 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:36 smithi077 bash[22596]: cluster 2024-09-13T00:37:36.157999+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2024-09-13T00:37:36.694 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:36 smithi077 bash[22596]: audit 2024-09-13T00:37:36.355033+0000 mon.b (mon.1) 76 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:36.694 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:36 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:36.694 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:37:36 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:36.694 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:37:36 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:36.695 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:37:36 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:36.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:36 smithi077 bash[32040]: cluster 2024-09-13T00:37:34.812779+0000 mgr.x (mgr.24490) 57 : cluster [DBG] pgmap v66: 137 pgs: 137 active+clean; 705 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 141 op/s 2024-09-13T00:37:36.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:36 smithi077 bash[32040]: cluster 2024-09-13T00:37:36.157999+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2024-09-13T00:37:36.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:36 smithi077 bash[32040]: audit 2024-09-13T00:37:36.355033+0000 mon.b (mon.1) 76 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:36.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:36 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:36.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:36 smithi088 bash[24895]: cluster 2024-09-13T00:37:34.812779+0000 mgr.x (mgr.24490) 57 : cluster [DBG] pgmap v66: 137 pgs: 137 active+clean; 705 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 141 op/s 2024-09-13T00:37:36.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:36 smithi088 bash[24895]: cluster 2024-09-13T00:37:36.157999+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e216: 8 total, 8 up, 8 in 2024-09-13T00:37:36.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:36 smithi088 bash[24895]: audit 2024-09-13T00:37:36.355033+0000 mon.b (mon.1) 76 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:37.014 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:36 smithi077 systemd[1]: Stopping Ceph mgr.y for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:37:37.617 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:37 smithi077 bash[72991]: ceph-9218306c-7165-11ef-bceb-c7b262605968-mgr-y 2024-09-13T00:37:37.617 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:37 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-13T00:37:37.617 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:37 smithi077 bash[73079]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-mgr-y 2024-09-13T00:37:37.617 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:37 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-09-13T00:37:37.617 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:37 smithi077 systemd[1]: Stopped Ceph mgr.y for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:37:37.810 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: after_prepare_snap 2024-09-13T00:37:37.882 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:37.883 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:37:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:37.883 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:37:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:37.884 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:37:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:37.884 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:37:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:37.885 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:37.885 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:37.885 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:37.886 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:37:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:37:38.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:37 smithi088 bash[53580]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:37:37] "GET /metrics HTTP/1.1" 200 38821 "" "Prometheus/2.43.0" 2024-09-13T00:37:38.150 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:37 smithi077 systemd[1]: Started Ceph mgr.y for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:37:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:38 smithi077 bash[22596]: cluster 2024-09-13T00:37:36.813726+0000 mgr.x (mgr.24490) 58 : cluster [DBG] pgmap v69: 137 pgs: 137 active+clean; 705 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:38 smithi077 bash[22596]: cluster 2024-09-13T00:37:37.157527+0000 mon.a (mon.0) 1102 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2024-09-13T00:37:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:38 smithi077 bash[22596]: audit 2024-09-13T00:37:37.497839+0000 mon.a (mon.0) 1103 : audit [DBG] from='client.? 172.21.15.77:0/2148432085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:38 smithi077 bash[22596]: audit 2024-09-13T00:37:37.923235+0000 mon.a (mon.0) 1104 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:38 smithi077 bash[22596]: audit 2024-09-13T00:37:37.933269+0000 mon.a (mon.0) 1105 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:38 smithi077 bash[22596]: audit 2024-09-13T00:37:37.937479+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:37:38.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:38 smithi077 bash[32040]: cluster 2024-09-13T00:37:36.813726+0000 mgr.x (mgr.24490) 58 : cluster [DBG] pgmap v69: 137 pgs: 137 active+clean; 705 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:38 smithi077 bash[32040]: cluster 2024-09-13T00:37:37.157527+0000 mon.a (mon.0) 1102 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2024-09-13T00:37:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:38 smithi077 bash[32040]: audit 2024-09-13T00:37:37.497839+0000 mon.a (mon.0) 1103 : audit [DBG] from='client.? 172.21.15.77:0/2148432085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:38 smithi077 bash[32040]: audit 2024-09-13T00:37:37.923235+0000 mon.a (mon.0) 1104 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:38 smithi077 bash[32040]: audit 2024-09-13T00:37:37.933269+0000 mon.a (mon.0) 1105 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:38 smithi077 bash[32040]: audit 2024-09-13T00:37:37.937479+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:37:38.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:38 smithi088 bash[24895]: cluster 2024-09-13T00:37:36.813726+0000 mgr.x (mgr.24490) 58 : cluster [DBG] pgmap v69: 137 pgs: 137 active+clean; 705 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:38.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:38 smithi088 bash[24895]: cluster 2024-09-13T00:37:37.157527+0000 mon.a (mon.0) 1102 : cluster [DBG] osdmap e217: 8 total, 8 up, 8 in 2024-09-13T00:37:38.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:38 smithi088 bash[24895]: audit 2024-09-13T00:37:37.497839+0000 mon.a (mon.0) 1103 : audit [DBG] from='client.? 172.21.15.77:0/2148432085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:38.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:38 smithi088 bash[24895]: audit 2024-09-13T00:37:37.923235+0000 mon.a (mon.0) 1104 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:38.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:38 smithi088 bash[24895]: audit 2024-09-13T00:37:37.933269+0000 mon.a (mon.0) 1105 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:38.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:38 smithi088 bash[24895]: audit 2024-09-13T00:37:37.937479+0000 mon.b (mon.1) 77 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:37:39.100 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:38 smithi088 bash[60003]: ts=2024-09-13T00:37:38.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:37:39.602 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 167772160 2024-09-13T00:37:39.626 INFO:tasks.workunit.client.0.smithi077.stdout:write: * 83886080~1000 2024-09-13T00:37:39.638 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:37:39.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:39 smithi077 bash[22596]: cluster 2024-09-13T00:37:38.574877+0000 mon.a (mon.0) 1106 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2024-09-13T00:37:39.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:39 smithi077 bash[73137]: debug 2024-09-13T00:37:39.472+0000 7f6c125be640 1 -- 172.21.15.77:0/2834735139 <== mon.0 v2:172.21.15.77:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559fa4110d00 con 0x559fa4103800 2024-09-13T00:37:39.776 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:39 smithi077 bash[73137]: debug 2024-09-13T00:37:39.592+0000 7f6c14e27040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T00:37:39.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:39 smithi077 bash[32040]: cluster 2024-09-13T00:37:38.574877+0000 mon.a (mon.0) 1106 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2024-09-13T00:37:39.841 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:37:39.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:39 smithi088 bash[24895]: cluster 2024-09-13T00:37:38.574877+0000 mon.a (mon.0) 1106 : cluster [DBG] osdmap e218: 8 total, 8 up, 8 in 2024-09-13T00:37:40.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:40 smithi088 bash[24895]: cluster 2024-09-13T00:37:38.814596+0000 mgr.x (mgr.24490) 59 : cluster [DBG] pgmap v72: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-09-13T00:37:40.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:40 smithi088 bash[24895]: cluster 2024-09-13T00:37:39.584593+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2024-09-13T00:37:40.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:40 smithi088 bash[24895]: audit 2024-09-13T00:37:39.905585+0000 mon.a (mon.0) 1108 : audit [DBG] from='client.? 172.21.15.77:0/2636956276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:40.862 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:40 smithi077 bash[32040]: cluster 2024-09-13T00:37:38.814596+0000 mgr.x (mgr.24490) 59 : cluster [DBG] pgmap v72: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-09-13T00:37:40.862 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:40 smithi077 bash[32040]: cluster 2024-09-13T00:37:39.584593+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2024-09-13T00:37:40.862 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:40 smithi077 bash[32040]: audit 2024-09-13T00:37:39.905585+0000 mon.a (mon.0) 1108 : audit [DBG] from='client.? 172.21.15.77:0/2636956276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:40.862 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:40 smithi077 bash[22596]: cluster 2024-09-13T00:37:38.814596+0000 mgr.x (mgr.24490) 59 : cluster [DBG] pgmap v72: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-09-13T00:37:40.863 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:40 smithi077 bash[22596]: cluster 2024-09-13T00:37:39.584593+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e219: 8 total, 8 up, 8 in 2024-09-13T00:37:40.863 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:40 smithi077 bash[22596]: audit 2024-09-13T00:37:39.905585+0000 mon.a (mon.0) 1108 : audit [DBG] from='client.? 172.21.15.77:0/2636956276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:41.276 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:40 smithi077 bash[73137]: debug 2024-09-13T00:37:40.864+0000 7f6c14e27040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T00:37:41.276 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:40 smithi077 bash[73137]: debug 2024-09-13T00:37:40.928+0000 7f6c14e27040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T00:37:41.276 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:41 smithi077 bash[73137]: debug 2024-09-13T00:37:41.076+0000 7f6c14e27040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T00:37:41.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:41 smithi088 bash[24895]: cluster 2024-09-13T00:37:40.592042+0000 mon.a (mon.0) 1109 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2024-09-13T00:37:41.866 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:41 smithi077 bash[22596]: cluster 2024-09-13T00:37:40.592042+0000 mon.a (mon.0) 1109 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2024-09-13T00:37:41.866 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:41 smithi077 bash[32040]: cluster 2024-09-13T00:37:40.592042+0000 mon.a (mon.0) 1109 : cluster [DBG] osdmap e220: 8 total, 8 up, 8 in 2024-09-13T00:37:41.866 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:41 smithi077 bash[73137]: debug 2024-09-13T00:37:41.728+0000 7f6c14e27040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T00:37:41.867 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:41 smithi077 bash[73137]: debug 2024-09-13T00:37:41.796+0000 7f6c14e27040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T00:37:42.129 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:41 smithi077 bash[73137]: debug 2024-09-13T00:37:41.868+0000 7f6c14e27040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T00:37:42.129 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:42 smithi077 bash[73137]: debug 2024-09-13T00:37:42.052+0000 7f6c14e27040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T00:37:42.129 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:42 smithi077 bash[73137]: debug 2024-09-13T00:37:42.128+0000 7f6c14e27040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T00:37:42.454 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:42 smithi077 bash[73137]: debug 2024-09-13T00:37:42.200+0000 7f6c14e27040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T00:37:42.454 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:42 smithi077 bash[73137]: debug 2024-09-13T00:37:42.456+0000 7f6c14e27040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T00:37:42.737 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:42 smithi077 bash[73137]: debug 2024-09-13T00:37:42.520+0000 7f6c14e27040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T00:37:42.738 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:42 smithi077 bash[22596]: cluster 2024-09-13T00:37:40.815108+0000 mgr.x (mgr.24490) 60 : cluster [DBG] pgmap v75: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-09-13T00:37:42.738 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:42 smithi077 bash[22596]: cluster 2024-09-13T00:37:41.602191+0000 mon.a (mon.0) 1110 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2024-09-13T00:37:42.738 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:42 smithi077 bash[22596]: audit 2024-09-13T00:37:42.248232+0000 mon.a (mon.0) 1111 : audit [DBG] from='client.? 172.21.15.77:0/551882325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:42.738 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:42 smithi077 bash[32040]: cluster 2024-09-13T00:37:40.815108+0000 mgr.x (mgr.24490) 60 : cluster [DBG] pgmap v75: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-09-13T00:37:42.738 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:42 smithi077 bash[32040]: cluster 2024-09-13T00:37:41.602191+0000 mon.a (mon.0) 1110 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2024-09-13T00:37:42.738 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:42 smithi077 bash[32040]: audit 2024-09-13T00:37:42.248232+0000 mon.a (mon.0) 1111 : audit [DBG] from='client.? 172.21.15.77:0/551882325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:42.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:42 smithi088 bash[24895]: cluster 2024-09-13T00:37:40.815108+0000 mgr.x (mgr.24490) 60 : cluster [DBG] pgmap v75: 137 pgs: 137 active+clean; 729 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-09-13T00:37:42.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:42 smithi088 bash[24895]: cluster 2024-09-13T00:37:41.602191+0000 mon.a (mon.0) 1110 : cluster [DBG] osdmap e221: 8 total, 8 up, 8 in 2024-09-13T00:37:42.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:42 smithi088 bash[24895]: audit 2024-09-13T00:37:42.248232+0000 mon.a (mon.0) 1111 : audit [DBG] from='client.? 172.21.15.77:0/551882325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:43.026 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:42 smithi077 bash[73137]: debug 2024-09-13T00:37:42.736+0000 7f6c14e27040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T00:37:43.026 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:42 smithi077 bash[73137]: debug 2024-09-13T00:37:42.832+0000 7f6c14e27040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T00:37:43.323 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.SnapsSnapExpandAfterPrepare (14173 ms) 2024-09-13T00:37:43.323 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.Clone 2024-09-13T00:37:43.372 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:37:43.389 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 0~8388609 2024-09-13T00:37:43.467 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 16777218~8388609 2024-09-13T00:37:43.510 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:43 smithi077 bash[73137]: debug 2024-09-13T00:37:43.136+0000 7f6c14e27040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T00:37:43.511 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:43 smithi077 bash[73137]: debug 2024-09-13T00:37:43.340+0000 7f6c14e27040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:37:43.543 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap 2024-09-13T00:37:43.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:43 smithi077 bash[32040]: cluster 2024-09-13T00:37:42.611194+0000 mon.a (mon.0) 1112 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2024-09-13T00:37:43.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:43 smithi077 bash[22596]: cluster 2024-09-13T00:37:42.611194+0000 mon.a (mon.0) 1112 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2024-09-13T00:37:43.777 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:43 smithi077 bash[72541]: ts=2024-09-13T00:37:43.515Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused" 2024-09-13T00:37:43.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:43 smithi077 bash[73137]: debug 2024-09-13T00:37:43.536+0000 7f6c14e27040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T00:37:43.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:43 smithi077 bash[73137]: debug 2024-09-13T00:37:43.620+0000 7f6c14e27040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T00:37:43.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:43 smithi088 bash[24895]: cluster 2024-09-13T00:37:42.611194+0000 mon.a (mon.0) 1112 : cluster [DBG] osdmap e222: 8 total, 8 up, 8 in 2024-09-13T00:37:43.850 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:43 smithi088 bash[60003]: ts=2024-09-13T00:37:43.506Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:37:44.276 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:43 smithi077 bash[73137]: debug 2024-09-13T00:37:43.804+0000 7f6c14e27040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:37:44.647 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap 2024-09-13T00:37:44.665 INFO:tasks.workunit.client.0.smithi077.stdout:clone image103 -> image106 2024-09-13T00:37:44.850 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:44 smithi088 bash[60003]: ts=2024-09-13T00:37:44.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:37:44.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:44 smithi088 bash[60003]: ts=2024-09-13T00:37:44.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:37:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:44 smithi088 bash[24895]: cluster 2024-09-13T00:37:42.815871+0000 mgr.x (mgr.24490) 61 : cluster [DBG] pgmap v78: 137 pgs: 137 active+clean; 749 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 502 op/s 2024-09-13T00:37:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:44 smithi088 bash[24895]: cluster 2024-09-13T00:37:43.622853+0000 mon.a (mon.0) 1113 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2024-09-13T00:37:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:44 smithi088 bash[24895]: audit 2024-09-13T00:37:43.820940+0000 mon.a (mon.0) 1114 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:44 smithi088 bash[24895]: audit 2024-09-13T00:37:43.830397+0000 mon.a (mon.0) 1115 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:44 smithi088 bash[24895]: audit 2024-09-13T00:37:44.554480+0000 mon.a (mon.0) 1116 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:44.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:44 smithi088 bash[24895]: audit 2024-09-13T00:37:44.565438+0000 mon.a (mon.0) 1117 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:44.861 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 0~1000 2024-09-13T00:37:44.955 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 1010~1000 2024-09-13T00:37:44.960 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap 2024-09-13T00:37:45.026 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:44 smithi077 bash[73137]: debug 2024-09-13T00:37:44.724+0000 7f6c14e27040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T00:37:45.026 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:44 smithi077 bash[73137]: debug 2024-09-13T00:37:44.808+0000 7f6c14e27040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T00:37:45.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:44 smithi077 bash[22596]: cluster 2024-09-13T00:37:42.815871+0000 mgr.x (mgr.24490) 61 : cluster [DBG] pgmap v78: 137 pgs: 137 active+clean; 749 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 502 op/s 2024-09-13T00:37:45.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:44 smithi077 bash[22596]: cluster 2024-09-13T00:37:43.622853+0000 mon.a (mon.0) 1113 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2024-09-13T00:37:45.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:44 smithi077 bash[22596]: audit 2024-09-13T00:37:43.820940+0000 mon.a (mon.0) 1114 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:45.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:44 smithi077 bash[22596]: audit 2024-09-13T00:37:43.830397+0000 mon.a (mon.0) 1115 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:45.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:44 smithi077 bash[22596]: audit 2024-09-13T00:37:44.554480+0000 mon.a (mon.0) 1116 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:45.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:44 smithi077 bash[22596]: audit 2024-09-13T00:37:44.565438+0000 mon.a (mon.0) 1117 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:45.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:44 smithi077 bash[32040]: cluster 2024-09-13T00:37:42.815871+0000 mgr.x (mgr.24490) 61 : cluster [DBG] pgmap v78: 137 pgs: 137 active+clean; 749 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 502 op/s 2024-09-13T00:37:45.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:44 smithi077 bash[32040]: cluster 2024-09-13T00:37:43.622853+0000 mon.a (mon.0) 1113 : cluster [DBG] osdmap e223: 8 total, 8 up, 8 in 2024-09-13T00:37:45.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:44 smithi077 bash[32040]: audit 2024-09-13T00:37:43.820940+0000 mon.a (mon.0) 1114 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:45.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:44 smithi077 bash[32040]: audit 2024-09-13T00:37:43.830397+0000 mon.a (mon.0) 1115 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:45.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:44 smithi077 bash[32040]: audit 2024-09-13T00:37:44.554480+0000 mon.a (mon.0) 1116 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:45.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:44 smithi077 bash[32040]: audit 2024-09-13T00:37:44.565438+0000 mon.a (mon.0) 1117 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:45.525 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[73137]: debug 2024-09-13T00:37:45.048+0000 7f6c14e27040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T00:37:45.526 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[73137]: debug 2024-09-13T00:37:45.124+0000 7f6c14e27040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T00:37:45.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[22596]: audit 2024-09-13T00:37:44.618671+0000 mon.c (mon.2) 171 : audit [DBG] from='client.? 172.21.15.77:0/1609695149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:45.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[22596]: cluster 2024-09-13T00:37:44.629587+0000 mon.a (mon.0) 1118 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2024-09-13T00:37:45.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[22596]: cluster 2024-09-13T00:37:44.816628+0000 mgr.x (mgr.24490) 62 : cluster [DBG] pgmap v81: 137 pgs: 137 active+clean; 749 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 502 op/s 2024-09-13T00:37:45.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[22596]: audit 2024-09-13T00:37:45.343657+0000 mon.a (mon.0) 1119 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:45.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[22596]: audit 2024-09-13T00:37:45.354607+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:45.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[32040]: audit 2024-09-13T00:37:44.618671+0000 mon.c (mon.2) 171 : audit [DBG] from='client.? 172.21.15.77:0/1609695149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:45.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[32040]: cluster 2024-09-13T00:37:44.629587+0000 mon.a (mon.0) 1118 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2024-09-13T00:37:45.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[32040]: cluster 2024-09-13T00:37:44.816628+0000 mgr.x (mgr.24490) 62 : cluster [DBG] pgmap v81: 137 pgs: 137 active+clean; 749 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 502 op/s 2024-09-13T00:37:45.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[32040]: audit 2024-09-13T00:37:45.343657+0000 mon.a (mon.0) 1119 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:45.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[32040]: audit 2024-09-13T00:37:45.354607+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:45.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[73137]: debug 2024-09-13T00:37:45.528+0000 7f6c14e27040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T00:37:45.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[73137]: debug 2024-09-13T00:37:45.608+0000 7f6c14e27040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T00:37:45.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[73137]: [13/Sep/2024:00:37:45] ENGINE Bus STARTING 2024-09-13T00:37:45.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[73137]: CherryPy Checker: 2024-09-13T00:37:45.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[73137]: The Application mounted at '' has an empty config. 2024-09-13T00:37:45.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[73137]: [13/Sep/2024:00:37:45] ENGINE Serving on http://:::9283 2024-09-13T00:37:45.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[73137]: [13/Sep/2024:00:37:45] ENGINE Bus STARTED 2024-09-13T00:37:46.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:45 smithi088 bash[24895]: audit 2024-09-13T00:37:44.618671+0000 mon.c (mon.2) 171 : audit [DBG] from='client.? 172.21.15.77:0/1609695149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:46.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:45 smithi088 bash[24895]: cluster 2024-09-13T00:37:44.629587+0000 mon.a (mon.0) 1118 : cluster [DBG] osdmap e224: 8 total, 8 up, 8 in 2024-09-13T00:37:46.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:45 smithi088 bash[24895]: cluster 2024-09-13T00:37:44.816628+0000 mgr.x (mgr.24490) 62 : cluster [DBG] pgmap v81: 137 pgs: 137 active+clean; 749 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 502 op/s 2024-09-13T00:37:46.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:45 smithi088 bash[24895]: audit 2024-09-13T00:37:45.343657+0000 mon.a (mon.0) 1119 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:46.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:45 smithi088 bash[24895]: audit 2024-09-13T00:37:45.354607+0000 mon.a (mon.0) 1120 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:46.195 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap 2024-09-13T00:37:46.226 INFO:tasks.workunit.client.0.smithi077.stdout:clone image106 -> image108 2024-09-13T00:37:46.276 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:37:45 smithi077 bash[72541]: ts=2024-09-13T00:37:45.919Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=4 2024-09-13T00:37:46.537 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 1000~1000 2024-09-13T00:37:46.638 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 2010~1000 2024-09-13T00:37:46.653 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:37:47.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:46 smithi077 bash[22596]: cluster 2024-09-13T00:37:45.636576+0000 mon.a (mon.0) 1121 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2024-09-13T00:37:47.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:46 smithi077 bash[22596]: audit 2024-09-13T00:37:45.637083+0000 mon.c (mon.2) 172 : audit [DBG] from='mgr.? 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T00:37:47.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:46 smithi077 bash[22596]: cluster 2024-09-13T00:37:45.637512+0000 mon.a (mon.0) 1122 : cluster [DBG] Standby manager daemon y restarted 2024-09-13T00:37:47.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:46 smithi077 bash[22596]: cluster 2024-09-13T00:37:45.637688+0000 mon.a (mon.0) 1123 : cluster [DBG] Standby manager daemon y started 2024-09-13T00:37:47.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:46 smithi077 bash[22596]: audit 2024-09-13T00:37:45.637964+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.? 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:37:47.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:46 smithi077 bash[22596]: audit 2024-09-13T00:37:45.640346+0000 mon.c (mon.2) 174 : audit [DBG] from='mgr.? 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T00:37:47.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:46 smithi077 bash[22596]: audit 2024-09-13T00:37:45.640744+0000 mon.c (mon.2) 175 : audit [DBG] from='mgr.? 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:37:47.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:46 smithi077 bash[22596]: cluster 2024-09-13T00:37:46.170709+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2024-09-13T00:37:47.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:46 smithi077 bash[32040]: cluster 2024-09-13T00:37:45.636576+0000 mon.a (mon.0) 1121 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2024-09-13T00:37:47.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:46 smithi077 bash[32040]: audit 2024-09-13T00:37:45.637083+0000 mon.c (mon.2) 172 : audit [DBG] from='mgr.? 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T00:37:47.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:46 smithi077 bash[32040]: cluster 2024-09-13T00:37:45.637512+0000 mon.a (mon.0) 1122 : cluster [DBG] Standby manager daemon y restarted 2024-09-13T00:37:47.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:46 smithi077 bash[32040]: cluster 2024-09-13T00:37:45.637688+0000 mon.a (mon.0) 1123 : cluster [DBG] Standby manager daemon y started 2024-09-13T00:37:47.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:46 smithi077 bash[32040]: audit 2024-09-13T00:37:45.637964+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.? 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:37:47.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:46 smithi077 bash[32040]: audit 2024-09-13T00:37:45.640346+0000 mon.c (mon.2) 174 : audit [DBG] from='mgr.? 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T00:37:47.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:46 smithi077 bash[32040]: audit 2024-09-13T00:37:45.640744+0000 mon.c (mon.2) 175 : audit [DBG] from='mgr.? 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:37:47.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:46 smithi077 bash[32040]: cluster 2024-09-13T00:37:46.170709+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2024-09-13T00:37:47.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:46 smithi088 bash[24895]: cluster 2024-09-13T00:37:45.636576+0000 mon.a (mon.0) 1121 : cluster [DBG] osdmap e225: 8 total, 8 up, 8 in 2024-09-13T00:37:47.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:46 smithi088 bash[24895]: audit 2024-09-13T00:37:45.637083+0000 mon.c (mon.2) 172 : audit [DBG] from='mgr.? 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-13T00:37:47.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:46 smithi088 bash[24895]: cluster 2024-09-13T00:37:45.637512+0000 mon.a (mon.0) 1122 : cluster [DBG] Standby manager daemon y restarted 2024-09-13T00:37:47.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:46 smithi088 bash[24895]: cluster 2024-09-13T00:37:45.637688+0000 mon.a (mon.0) 1123 : cluster [DBG] Standby manager daemon y started 2024-09-13T00:37:47.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:46 smithi088 bash[24895]: audit 2024-09-13T00:37:45.637964+0000 mon.c (mon.2) 173 : audit [DBG] from='mgr.? 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:37:47.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:46 smithi088 bash[24895]: audit 2024-09-13T00:37:45.640346+0000 mon.c (mon.2) 174 : audit [DBG] from='mgr.? 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-13T00:37:47.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:46 smithi088 bash[24895]: audit 2024-09-13T00:37:45.640744+0000 mon.c (mon.2) 175 : audit [DBG] from='mgr.? 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:37:47.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:46 smithi088 bash[24895]: cluster 2024-09-13T00:37:46.170709+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e226: 8 total, 8 up, 8 in 2024-09-13T00:37:47.304 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:37:47.757 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:37:48.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:47 smithi077 bash[22596]: cluster 2024-09-13T00:37:46.650896+0000 mon.a (mon.0) 1125 : cluster [DBG] mgrmap e26: x(active, since 55s), standbys: y 2024-09-13T00:37:48.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:47 smithi077 bash[22596]: cluster 2024-09-13T00:37:46.817477+0000 mgr.x (mgr.24490) 63 : cluster [DBG] pgmap v84: 137 pgs: 137 active+clean; 749 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:48.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:47 smithi077 bash[22596]: audit 2024-09-13T00:37:46.992523+0000 mon.c (mon.2) 176 : audit [DBG] from='client.? 172.21.15.77:0/2768031530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:48.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:47 smithi077 bash[32040]: cluster 2024-09-13T00:37:46.650896+0000 mon.a (mon.0) 1125 : cluster [DBG] mgrmap e26: x(active, since 55s), standbys: y 2024-09-13T00:37:48.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:47 smithi077 bash[32040]: cluster 2024-09-13T00:37:46.817477+0000 mgr.x (mgr.24490) 63 : cluster [DBG] pgmap v84: 137 pgs: 137 active+clean; 749 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:48.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:47 smithi077 bash[32040]: audit 2024-09-13T00:37:46.992523+0000 mon.c (mon.2) 176 : audit [DBG] from='client.? 172.21.15.77:0/2768031530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:48.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:47 smithi088 bash[53580]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:37:47] "GET /metrics HTTP/1.1" 200 38821 "" "Prometheus/2.43.0" 2024-09-13T00:37:48.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:47 smithi088 bash[24895]: cluster 2024-09-13T00:37:46.650896+0000 mon.a (mon.0) 1125 : cluster [DBG] mgrmap e26: x(active, since 55s), standbys: y 2024-09-13T00:37:48.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:47 smithi088 bash[24895]: cluster 2024-09-13T00:37:46.817477+0000 mgr.x (mgr.24490) 63 : cluster [DBG] pgmap v84: 137 pgs: 137 active+clean; 749 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:48.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:47 smithi088 bash[24895]: audit 2024-09-13T00:37:46.992523+0000 mon.c (mon.2) 176 : audit [DBG] from='client.? 172.21.15.77:0/2768031530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:48.191 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.Clone (4868 ms) 2024-09-13T00:37:48.191 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.CloneParent 2024-09-13T00:37:48.245 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:37:48.267 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap 2024-09-13T00:37:49.100 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:48 smithi088 bash[60003]: ts=2024-09-13T00:37:48.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:37:49.711 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap 2024-09-13T00:37:49.733 INFO:tasks.workunit.client.0.smithi077.stdout:clone image109 -> image112 2024-09-13T00:37:49.968 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:37:50.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:49 smithi077 bash[22596]: cluster 2024-09-13T00:37:48.678261+0000 mon.a (mon.0) 1126 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2024-09-13T00:37:50.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:49 smithi077 bash[22596]: cluster 2024-09-13T00:37:48.818443+0000 mgr.x (mgr.24490) 64 : cluster [DBG] pgmap v86: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 11 MiB/s wr, 533 op/s 2024-09-13T00:37:50.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:49 smithi077 bash[22596]: audit 2024-09-13T00:37:49.353326+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.77:0/2696602753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:50.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:49 smithi077 bash[32040]: cluster 2024-09-13T00:37:48.678261+0000 mon.a (mon.0) 1126 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2024-09-13T00:37:50.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:49 smithi077 bash[32040]: cluster 2024-09-13T00:37:48.818443+0000 mgr.x (mgr.24490) 64 : cluster [DBG] pgmap v86: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 11 MiB/s wr, 533 op/s 2024-09-13T00:37:50.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:49 smithi077 bash[32040]: audit 2024-09-13T00:37:49.353326+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.77:0/2696602753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:50.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:49 smithi088 bash[24895]: cluster 2024-09-13T00:37:48.678261+0000 mon.a (mon.0) 1126 : cluster [DBG] osdmap e227: 8 total, 8 up, 8 in 2024-09-13T00:37:50.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:49 smithi088 bash[24895]: cluster 2024-09-13T00:37:48.818443+0000 mgr.x (mgr.24490) 64 : cluster [DBG] pgmap v86: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 50 MiB/s rd, 11 MiB/s wr, 533 op/s 2024-09-13T00:37:50.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:49 smithi088 bash[24895]: audit 2024-09-13T00:37:49.353326+0000 mon.c (mon.2) 177 : audit [DBG] from='client.? 172.21.15.77:0/2696602753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:50.320 INFO:tasks.workunit.client.0.smithi077.stdout:migration_abort 2024-09-13T00:37:50.643 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.CloneParent (2453 ms) 2024-09-13T00:37:50.644 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.CloneUpdateAfterPrepare 2024-09-13T00:37:50.727 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:37:50.753 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 0~10 2024-09-13T00:37:50.775 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap 2024-09-13T00:37:51.026 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:50 smithi077 bash[22596]: cluster 2024-09-13T00:37:49.691813+0000 mon.a (mon.0) 1127 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2024-09-13T00:37:51.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:50 smithi077 bash[32040]: cluster 2024-09-13T00:37:49.691813+0000 mon.a (mon.0) 1127 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2024-09-13T00:37:51.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:50 smithi088 bash[24895]: cluster 2024-09-13T00:37:49.691813+0000 mon.a (mon.0) 1127 : cluster [DBG] osdmap e228: 8 total, 8 up, 8 in 2024-09-13T00:37:52.201 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap 2024-09-13T00:37:52.226 INFO:tasks.workunit.client.0.smithi077.stdout:clone image113 -> image116 2024-09-13T00:37:52.472 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:37:52.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:52 smithi077 bash[22596]: cluster 2024-09-13T00:37:50.819264+0000 mgr.x (mgr.24490) 65 : cluster [DBG] pgmap v88: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 9.3 MiB/s wr, 430 op/s 2024-09-13T00:37:52.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:52 smithi077 bash[22596]: cluster 2024-09-13T00:37:51.170583+0000 mon.a (mon.0) 1128 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2024-09-13T00:37:52.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:52 smithi077 bash[22596]: audit 2024-09-13T00:37:51.374849+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:52.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:52 smithi077 bash[22596]: audit 2024-09-13T00:37:51.728219+0000 mon.c (mon.2) 178 : audit [DBG] from='client.? 172.21.15.77:0/1477396182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:52.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:52 smithi077 bash[32040]: cluster 2024-09-13T00:37:50.819264+0000 mgr.x (mgr.24490) 65 : cluster [DBG] pgmap v88: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 9.3 MiB/s wr, 430 op/s 2024-09-13T00:37:52.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:52 smithi077 bash[32040]: cluster 2024-09-13T00:37:51.170583+0000 mon.a (mon.0) 1128 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2024-09-13T00:37:52.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:52 smithi077 bash[32040]: audit 2024-09-13T00:37:51.374849+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:52.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:52 smithi077 bash[32040]: audit 2024-09-13T00:37:51.728219+0000 mon.c (mon.2) 178 : audit [DBG] from='client.? 172.21.15.77:0/1477396182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:52.600 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:52 smithi088 bash[60003]: ts=2024-09-13T00:37:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:37:52.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:52 smithi088 bash[24895]: cluster 2024-09-13T00:37:50.819264+0000 mgr.x (mgr.24490) 65 : cluster [DBG] pgmap v88: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 40 MiB/s rd, 9.3 MiB/s wr, 430 op/s 2024-09-13T00:37:52.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:52 smithi088 bash[24895]: cluster 2024-09-13T00:37:51.170583+0000 mon.a (mon.0) 1128 : cluster [DBG] osdmap e229: 8 total, 8 up, 8 in 2024-09-13T00:37:52.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:52 smithi088 bash[24895]: audit 2024-09-13T00:37:51.374849+0000 mon.b (mon.1) 78 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:52.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:52 smithi088 bash[24895]: audit 2024-09-13T00:37:51.728219+0000 mon.c (mon.2) 178 : audit [DBG] from='client.? 172.21.15.77:0/1477396182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:52.844 INFO:tasks.workunit.client.0.smithi077.stdout:write: Y 0~1 2024-09-13T00:37:52.897 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:37:53.081 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:37:53.276 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:53 smithi077 bash[32040]: cluster 2024-09-13T00:37:52.183735+0000 mon.a (mon.0) 1129 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2024-09-13T00:37:53.276 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:53 smithi077 bash[22596]: cluster 2024-09-13T00:37:52.183735+0000 mon.a (mon.0) 1129 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2024-09-13T00:37:53.338 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.CloneUpdateAfterPrepare (2694 ms) 2024-09-13T00:37:53.339 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.TriggerAssertSnapcSeq 2024-09-13T00:37:53.418 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:37:53.448 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 41943040~10 2024-09-13T00:37:53.468 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:37:53.506 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:53 smithi088 bash[24895]: cluster 2024-09-13T00:37:52.183735+0000 mon.a (mon.0) 1129 : cluster [DBG] osdmap e230: 8 total, 8 up, 8 in 2024-09-13T00:37:53.850 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:53 smithi088 bash[60003]: ts=2024-09-13T00:37:53.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:37:54.451 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:54 smithi088 bash[24895]: cluster 2024-09-13T00:37:52.820132+0000 mgr.x (mgr.24490) 66 : cluster [DBG] pgmap v91: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 9.2 MiB/s rd, 1010 KiB/s wr, 684 op/s 2024-09-13T00:37:54.452 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:54 smithi088 bash[24895]: audit 2024-09-13T00:37:54.089342+0000 mon.a (mon.0) 1130 : audit [DBG] from='client.? 172.21.15.77:0/75217975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:54.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:54 smithi077 bash[22596]: cluster 2024-09-13T00:37:52.820132+0000 mgr.x (mgr.24490) 66 : cluster [DBG] pgmap v91: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 9.2 MiB/s rd, 1010 KiB/s wr, 684 op/s 2024-09-13T00:37:54.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:54 smithi077 bash[22596]: audit 2024-09-13T00:37:54.089342+0000 mon.a (mon.0) 1130 : audit [DBG] from='client.? 172.21.15.77:0/75217975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:54.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:54 smithi077 bash[32040]: cluster 2024-09-13T00:37:52.820132+0000 mgr.x (mgr.24490) 66 : cluster [DBG] pgmap v91: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 9.2 MiB/s rd, 1010 KiB/s wr, 684 op/s 2024-09-13T00:37:54.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:54 smithi077 bash[32040]: audit 2024-09-13T00:37:54.089342+0000 mon.a (mon.0) 1130 : audit [DBG] from='client.? 172.21.15.77:0/75217975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:54.850 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:54 smithi088 bash[60003]: ts=2024-09-13T00:37:54.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:37:54.850 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:37:54 smithi088 bash[60003]: ts=2024-09-13T00:37:54.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:37:55.224 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 41943041~10 2024-09-13T00:37:55.238 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:37:55.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:55 smithi077 bash[32040]: cluster 2024-09-13T00:37:54.206700+0000 mon.a (mon.0) 1131 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2024-09-13T00:37:55.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:55 smithi077 bash[22596]: cluster 2024-09-13T00:37:54.206700+0000 mon.a (mon.0) 1131 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2024-09-13T00:37:55.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:55 smithi088 bash[24895]: cluster 2024-09-13T00:37:54.206700+0000 mon.a (mon.0) 1131 : cluster [DBG] osdmap e231: 8 total, 8 up, 8 in 2024-09-13T00:37:56.413 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 41943042~10 2024-09-13T00:37:56.441 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap2 2024-09-13T00:37:56.526 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[73137]: [13/Sep/2024:00:37:56] ENGINE Bus STOPPING 2024-09-13T00:37:56.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[32040]: cluster 2024-09-13T00:37:54.820978+0000 mgr.x (mgr.24490) 67 : cluster [DBG] pgmap v93: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 7.4 MiB/s rd, 815 KiB/s wr, 552 op/s 2024-09-13T00:37:56.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[32040]: cluster 2024-09-13T00:37:55.212953+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2024-09-13T00:37:56.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[32040]: audit 2024-09-13T00:37:56.076433+0000 mon.a (mon.0) 1133 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:56.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[32040]: audit 2024-09-13T00:37:56.089269+0000 mon.a (mon.0) 1134 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:56.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[32040]: audit 2024-09-13T00:37:56.091336+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:37:56.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[32040]: audit 2024-09-13T00:37:56.093022+0000 mon.b (mon.1) 80 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:37:56.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[32040]: audit 2024-09-13T00:37:56.103971+0000 mon.a (mon.0) 1135 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:56.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[22596]: cluster 2024-09-13T00:37:54.820978+0000 mgr.x (mgr.24490) 67 : cluster [DBG] pgmap v93: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 7.4 MiB/s rd, 815 KiB/s wr, 552 op/s 2024-09-13T00:37:56.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[22596]: cluster 2024-09-13T00:37:55.212953+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2024-09-13T00:37:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[22596]: audit 2024-09-13T00:37:56.076433+0000 mon.a (mon.0) 1133 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[22596]: audit 2024-09-13T00:37:56.089269+0000 mon.a (mon.0) 1134 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[22596]: audit 2024-09-13T00:37:56.091336+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:37:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[22596]: audit 2024-09-13T00:37:56.093022+0000 mon.b (mon.1) 80 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:37:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[22596]: audit 2024-09-13T00:37:56.103971+0000 mon.a (mon.0) 1135 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:56.600 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:56 smithi088 bash[53580]: ignoring --setuser ceph since I am not root 2024-09-13T00:37:56.600 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:56 smithi088 bash[53580]: ignoring --setgroup ceph since I am not root 2024-09-13T00:37:56.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:56 smithi088 bash[53580]: debug 2024-09-13T00:37:56.310+0000 7fba227e3640 1 -- 172.21.15.88:0/3206193135 <== mon.1 v2:172.21.15.88:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fdcb2b8d00 con 0x55fdcb2aa800 2024-09-13T00:37:56.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:56 smithi088 bash[53580]: debug 2024-09-13T00:37:56.422+0000 7fba2503e040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T00:37:56.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:56 smithi088 bash[24895]: cluster 2024-09-13T00:37:54.820978+0000 mgr.x (mgr.24490) 67 : cluster [DBG] pgmap v93: 137 pgs: 137 active+clean; 777 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 7.4 MiB/s rd, 815 KiB/s wr, 552 op/s 2024-09-13T00:37:56.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:56 smithi088 bash[24895]: cluster 2024-09-13T00:37:55.212953+0000 mon.a (mon.0) 1132 : cluster [DBG] osdmap e232: 8 total, 8 up, 8 in 2024-09-13T00:37:56.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:56 smithi088 bash[24895]: audit 2024-09-13T00:37:56.076433+0000 mon.a (mon.0) 1133 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:56.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:56 smithi088 bash[24895]: audit 2024-09-13T00:37:56.089269+0000 mon.a (mon.0) 1134 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:56.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:56 smithi088 bash[24895]: audit 2024-09-13T00:37:56.091336+0000 mon.b (mon.1) 79 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:37:56.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:56 smithi088 bash[24895]: audit 2024-09-13T00:37:56.093022+0000 mon.b (mon.1) 80 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:37:56.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:56 smithi088 bash[24895]: audit 2024-09-13T00:37:56.103971+0000 mon.a (mon.0) 1135 : audit [INF] from='mgr.24490 ' entity='mgr.x' 2024-09-13T00:37:57.026 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[73137]: [13/Sep/2024:00:37:56] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:37:57.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[73137]: [13/Sep/2024:00:37:56] ENGINE Bus STOPPED 2024-09-13T00:37:57.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[73137]: [13/Sep/2024:00:37:56] ENGINE Bus STARTING 2024-09-13T00:37:57.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[73137]: debug 2024-09-13T00:37:56.908+0000 7f6b7f19c640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:37:57.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[73137]: debug 2024-09-13T00:37:56.908+0000 7f6b7f19c640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:37:57.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[73137]: debug 2024-09-13T00:37:56.908+0000 7f6b7f19c640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:37:57.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[73137]: debug 2024-09-13T00:37:56.908+0000 7f6b7f19c640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:37:57.028 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[73137]: debug 2024-09-13T00:37:56.908+0000 7f6b7f19c640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:37:57.028 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[73137]: debug 2024-09-13T00:37:56.912+0000 7f6b7c196640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:37:57.028 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[73137]: debug 2024-09-13T00:37:56.912+0000 7f6b7c196640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:37:57.028 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[73137]: debug 2024-09-13T00:37:56.912+0000 7f6b7c196640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:37:57.029 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[73137]: debug 2024-09-13T00:37:56.912+0000 7f6b7c196640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:37:57.029 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[73137]: debug 2024-09-13T00:37:56.912+0000 7f6b7c196640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:37:57.029 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[73137]: [13/Sep/2024:00:37:56] ENGINE Serving on http://:::9283 2024-09-13T00:37:57.029 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:56 smithi077 bash[73137]: [13/Sep/2024:00:37:56] ENGINE Bus STARTED 2024-09-13T00:37:57.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: cluster 2024-09-13T00:37:56.220668+0000 mon.a (mon.0) 1136 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2024-09-13T00:37:57.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.229423+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:37:57.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.238341+0000 mon.b (mon.1) 82 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T00:37:57.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.239785+0000 mon.a (mon.0) 1137 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T00:37:57.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: cluster 2024-09-13T00:37:56.240337+0000 mon.a (mon.0) 1138 : cluster [INF] Activating manager daemon y 2024-09-13T00:37:57.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: cluster 2024-09-13T00:37:56.246243+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2024-09-13T00:37:57.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.248395+0000 mon.a (mon.0) 1140 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T00:37:57.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: cluster 2024-09-13T00:37:56.248475+0000 mon.a (mon.0) 1141 : cluster [DBG] mgrmap e27: y(active, starting, since 0.0081355s) 2024-09-13T00:37:57.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.255527+0000 mon.c (mon.2) 179 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:37:57.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.255956+0000 mon.c (mon.2) 180 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:37:57.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.256363+0000 mon.c (mon.2) 181 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:37:57.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.258280+0000 mon.c (mon.2) 182 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi077.bigbbf"}]: dispatch 2024-09-13T00:37:57.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.258845+0000 mon.c (mon.2) 183 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi088.krmowh"}]: dispatch 2024-09-13T00:37:57.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.259322+0000 mon.c (mon.2) 184 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T00:37:57.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.259736+0000 mon.c (mon.2) 185 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:37:57.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.260175+0000 mon.c (mon.2) 186 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:37:57.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.260533+0000 mon.c (mon.2) 187 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:37:57.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.261027+0000 mon.c (mon.2) 188 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:37:57.355 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.261516+0000 mon.c (mon.2) 189 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:37:57.355 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.262002+0000 mon.c (mon.2) 190 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:37:57.355 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.262506+0000 mon.c (mon.2) 191 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:37:57.355 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.262987+0000 mon.c (mon.2) 192 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:37:57.356 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.263482+0000 mon.c (mon.2) 193 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T00:37:57.356 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.264041+0000 mon.c (mon.2) 194 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T00:37:57.356 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.264982+0000 mon.c (mon.2) 195 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T00:37:57.356 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.498962+0000 mon.a (mon.0) 1142 : audit [DBG] from='client.? 172.21.15.77:0/1192721232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:57.356 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: cluster 2024-09-13T00:37:56.794455+0000 mon.a (mon.0) 1143 : cluster [INF] Manager daemon y is now available 2024-09-13T00:37:57.357 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.819923+0000 mon.c (mon.2) 196 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:37:57.357 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.847316+0000 mon.c (mon.2) 197 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:57.357 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.856453+0000 mon.c (mon.2) 198 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:37:57.357 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.856945+0000 mon.a (mon.0) 1144 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:37:57.357 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.892020+0000 mon.c (mon.2) 199 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:37:57.357 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[24895]: audit 2024-09-13T00:37:56.892522+0000 mon.a (mon.0) 1145 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:37:57.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: cluster 2024-09-13T00:37:56.220668+0000 mon.a (mon.0) 1136 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2024-09-13T00:37:57.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.229423+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:37:57.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.238341+0000 mon.b (mon.1) 82 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T00:37:57.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.239785+0000 mon.a (mon.0) 1137 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T00:37:57.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: cluster 2024-09-13T00:37:56.240337+0000 mon.a (mon.0) 1138 : cluster [INF] Activating manager daemon y 2024-09-13T00:37:57.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: cluster 2024-09-13T00:37:56.246243+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2024-09-13T00:37:57.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.248395+0000 mon.a (mon.0) 1140 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T00:37:57.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: cluster 2024-09-13T00:37:56.248475+0000 mon.a (mon.0) 1141 : cluster [DBG] mgrmap e27: y(active, starting, since 0.0081355s) 2024-09-13T00:37:57.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.255527+0000 mon.c (mon.2) 179 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:37:57.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.255956+0000 mon.c (mon.2) 180 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:37:57.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.256363+0000 mon.c (mon.2) 181 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:37:57.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.258280+0000 mon.c (mon.2) 182 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi077.bigbbf"}]: dispatch 2024-09-13T00:37:57.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.258845+0000 mon.c (mon.2) 183 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi088.krmowh"}]: dispatch 2024-09-13T00:37:57.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.259322+0000 mon.c (mon.2) 184 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T00:37:57.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.259736+0000 mon.c (mon.2) 185 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:37:57.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.260175+0000 mon.c (mon.2) 186 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:37:57.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.260533+0000 mon.c (mon.2) 187 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:37:57.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.261027+0000 mon.c (mon.2) 188 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:37:57.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.261516+0000 mon.c (mon.2) 189 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:37:57.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: cluster 2024-09-13T00:37:56.220668+0000 mon.a (mon.0) 1136 : cluster [DBG] osdmap e233: 8 total, 8 up, 8 in 2024-09-13T00:37:57.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.229423+0000 mon.b (mon.1) 81 : audit [DBG] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:37:57.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.238341+0000 mon.b (mon.1) 82 : audit [INF] from='mgr.24490 172.21.15.88:0/3833379757' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T00:37:57.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.239785+0000 mon.a (mon.0) 1137 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-13T00:37:57.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: cluster 2024-09-13T00:37:56.240337+0000 mon.a (mon.0) 1138 : cluster [INF] Activating manager daemon y 2024-09-13T00:37:57.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: cluster 2024-09-13T00:37:56.246243+0000 mon.a (mon.0) 1139 : cluster [DBG] osdmap e234: 8 total, 8 up, 8 in 2024-09-13T00:37:57.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.248395+0000 mon.a (mon.0) 1140 : audit [INF] from='mgr.24490 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-13T00:37:57.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: cluster 2024-09-13T00:37:56.248475+0000 mon.a (mon.0) 1141 : cluster [DBG] mgrmap e27: y(active, starting, since 0.0081355s) 2024-09-13T00:37:57.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.255527+0000 mon.c (mon.2) 179 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:37:57.533 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.255956+0000 mon.c (mon.2) 180 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:37:57.533 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.256363+0000 mon.c (mon.2) 181 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:37:57.533 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.258280+0000 mon.c (mon.2) 182 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi077.bigbbf"}]: dispatch 2024-09-13T00:37:57.533 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.258845+0000 mon.c (mon.2) 183 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi088.krmowh"}]: dispatch 2024-09-13T00:37:57.533 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.259322+0000 mon.c (mon.2) 184 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T00:37:57.533 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.259736+0000 mon.c (mon.2) 185 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:37:57.534 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.260175+0000 mon.c (mon.2) 186 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:37:57.534 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.260533+0000 mon.c (mon.2) 187 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:37:57.534 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.261027+0000 mon.c (mon.2) 188 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:37:57.534 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.261516+0000 mon.c (mon.2) 189 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:37:57.534 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.262002+0000 mon.c (mon.2) 190 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:37:57.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.262506+0000 mon.c (mon.2) 191 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:37:57.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.262987+0000 mon.c (mon.2) 192 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:37:57.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.263482+0000 mon.c (mon.2) 193 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T00:37:57.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.264041+0000 mon.c (mon.2) 194 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T00:37:57.535 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.264982+0000 mon.c (mon.2) 195 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T00:37:57.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.498962+0000 mon.a (mon.0) 1142 : audit [DBG] from='client.? 172.21.15.77:0/1192721232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:57.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: cluster 2024-09-13T00:37:56.794455+0000 mon.a (mon.0) 1143 : cluster [INF] Manager daemon y is now available 2024-09-13T00:37:57.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.819923+0000 mon.c (mon.2) 196 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:37:57.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.847316+0000 mon.c (mon.2) 197 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:57.536 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.856453+0000 mon.c (mon.2) 198 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:37:57.537 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.856945+0000 mon.a (mon.0) 1144 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:37:57.537 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.892020+0000 mon.c (mon.2) 199 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:37:57.537 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[32040]: audit 2024-09-13T00:37:56.892522+0000 mon.a (mon.0) 1145 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:37:57.537 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.262002+0000 mon.c (mon.2) 190 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:37:57.538 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.262506+0000 mon.c (mon.2) 191 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:37:57.538 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.262987+0000 mon.c (mon.2) 192 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:37:57.538 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.263482+0000 mon.c (mon.2) 193 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T00:37:57.538 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.264041+0000 mon.c (mon.2) 194 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T00:37:57.538 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.264982+0000 mon.c (mon.2) 195 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T00:37:57.538 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.498962+0000 mon.a (mon.0) 1142 : audit [DBG] from='client.? 172.21.15.77:0/1192721232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:57.539 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: cluster 2024-09-13T00:37:56.794455+0000 mon.a (mon.0) 1143 : cluster [INF] Manager daemon y is now available 2024-09-13T00:37:57.539 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.819923+0000 mon.c (mon.2) 196 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:37:57.539 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.847316+0000 mon.c (mon.2) 197 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:37:57.539 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.856453+0000 mon.c (mon.2) 198 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:37:57.539 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.856945+0000 mon.a (mon.0) 1144 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:37:57.540 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.892020+0000 mon.c (mon.2) 199 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:37:57.540 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:57 smithi077 bash[22596]: audit 2024-09-13T00:37:56.892522+0000 mon.a (mon.0) 1145 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:37:58.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[53580]: debug 2024-09-13T00:37:57.662+0000 7fba2503e040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T00:37:58.101 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[53580]: debug 2024-09-13T00:37:57.730+0000 7fba2503e040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T00:37:58.101 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:57 smithi088 bash[53580]: debug 2024-09-13T00:37:57.878+0000 7fba2503e040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T00:37:58.288 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 41943040 2024-09-13T00:37:58.359 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:37:58.511 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:37:58.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[22596]: cluster 2024-09-13T00:37:57.245986+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2024-09-13T00:37:58.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[22596]: cluster 2024-09-13T00:37:57.251732+0000 mon.a (mon.0) 1147 : cluster [DBG] mgrmap e28: y(active, since 1.01139s) 2024-09-13T00:37:58.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[22596]: cluster 2024-09-13T00:37:57.294218+0000 mgr.y (mgr.25064) 1 : cluster [DBG] pgmap v4: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:58.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[32040]: cluster 2024-09-13T00:37:57.245986+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2024-09-13T00:37:58.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[32040]: cluster 2024-09-13T00:37:57.251732+0000 mon.a (mon.0) 1147 : cluster [DBG] mgrmap e28: y(active, since 1.01139s) 2024-09-13T00:37:58.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[32040]: cluster 2024-09-13T00:37:57.294218+0000 mgr.y (mgr.25064) 1 : cluster [DBG] pgmap v4: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:58.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: debug 2024-09-13T00:37:58.083+0000 7f6b6896f640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:00:37:58] ENGINE Error in HTTPServer.serve 2024-09-13T00:37:58.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: Traceback (most recent call last): 2024-09-13T00:37:58.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:37:58.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: self._connections.run(self.expiration_interval) 2024-09-13T00:37:58.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:37:58.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: self._run(expiration_interval) 2024-09-13T00:37:58.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:37:58.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:37:58.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:37:58.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:37:58.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:37:58.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: s = self.context.wrap_socket( 2024-09-13T00:37:58.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:37:58.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: return self.sslsocket_class._create( 2024-09-13T00:37:58.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:37:58.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: self.do_handshake() 2024-09-13T00:37:58.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:37:58.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: self._sslobj.do_handshake() 2024-09-13T00:37:58.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:37:58 smithi077 bash[73137]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:37:58.559 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:58 smithi088 bash[24895]: cluster 2024-09-13T00:37:57.245986+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e235: 8 total, 8 up, 8 in 2024-09-13T00:37:58.559 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:58 smithi088 bash[24895]: cluster 2024-09-13T00:37:57.251732+0000 mon.a (mon.0) 1147 : cluster [DBG] mgrmap e28: y(active, since 1.01139s) 2024-09-13T00:37:58.559 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:58 smithi088 bash[24895]: cluster 2024-09-13T00:37:57.294218+0000 mgr.y (mgr.25064) 1 : cluster [DBG] pgmap v4: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:58.850 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:58 smithi088 bash[53580]: debug 2024-09-13T00:37:58.558+0000 7fba2503e040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T00:37:58.850 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:58 smithi088 bash[53580]: debug 2024-09-13T00:37:58.630+0000 7fba2503e040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T00:37:58.850 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:58 smithi088 bash[53580]: debug 2024-09-13T00:37:58.698+0000 7fba2503e040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T00:37:59.268 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:58 smithi088 bash[53580]: debug 2024-09-13T00:37:58.886+0000 7fba2503e040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T00:37:59.268 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:58 smithi088 bash[53580]: debug 2024-09-13T00:37:58.962+0000 7fba2503e040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T00:37:59.268 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[53580]: debug 2024-09-13T00:37:59.030+0000 7fba2503e040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T00:37:59.497 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.TriggerAssertSnapcSeq (6159 ms) 2024-09-13T00:37:59.497 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.SnapTrimBeforePrepare 2024-09-13T00:37:59.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: cephadm 2024-09-13T00:37:57.964753+0000 mgr.y (mgr.25064) 2 : cephadm [INF] [13/Sep/2024:00:37:57] ENGINE Bus STARTING 2024-09-13T00:37:59.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: cephadm 2024-09-13T00:37:58.084168+0000 mgr.y (mgr.25064) 3 : cephadm [INF] [13/Sep/2024:00:37:58] ENGINE Serving on https://172.21.15.77:7150 2024-09-13T00:37:59.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: cephadm 2024-09-13T00:37:58.089166+0000 mgr.y (mgr.25064) 4 : cephadm [ERR] [13/Sep/2024:00:37:58] ENGINE Error in HTTPServer.serve 2024-09-13T00:37:59.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: Traceback (most recent call last): 2024-09-13T00:37:59.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:37:59.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: self._connections.run(self.expiration_interval) 2024-09-13T00:37:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:37:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: self._run(expiration_interval) 2024-09-13T00:37:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:37:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:37:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:37:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:37:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:37:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: s = self.context.wrap_socket( 2024-09-13T00:37:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:37:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: return self.sslsocket_class._create( 2024-09-13T00:37:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:37:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: self.do_handshake() 2024-09-13T00:37:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:37:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: self._sslobj.do_handshake() 2024-09-13T00:37:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:37:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: cephadm 2024-09-13T00:37:58.185253+0000 mgr.y (mgr.25064) 5 : cephadm [INF] [13/Sep/2024:00:37:58] ENGINE Serving on http://172.21.15.77:8765 2024-09-13T00:37:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: cephadm 2024-09-13T00:37:58.185340+0000 mgr.y (mgr.25064) 6 : cephadm [INF] [13/Sep/2024:00:37:58] ENGINE Bus STARTED 2024-09-13T00:37:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: cluster 2024-09-13T00:37:58.259261+0000 mgr.y (mgr.25064) 7 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: cluster 2024-09-13T00:37:58.266790+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2024-09-13T00:37:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: cluster 2024-09-13T00:37:58.267391+0000 mon.a (mon.0) 1149 : cluster [DBG] mgrmap e29: y(active, since 2s) 2024-09-13T00:37:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[22596]: audit 2024-09-13T00:37:58.863523+0000 mon.a (mon.0) 1150 : audit [DBG] from='client.? 172.21.15.77:0/3926927439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:59.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: cephadm 2024-09-13T00:37:57.964753+0000 mgr.y (mgr.25064) 2 : cephadm [INF] [13/Sep/2024:00:37:57] ENGINE Bus STARTING 2024-09-13T00:37:59.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: cephadm 2024-09-13T00:37:58.084168+0000 mgr.y (mgr.25064) 3 : cephadm [INF] [13/Sep/2024:00:37:58] ENGINE Serving on https://172.21.15.77:7150 2024-09-13T00:37:59.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: cephadm 2024-09-13T00:37:58.089166+0000 mgr.y (mgr.25064) 4 : cephadm [ERR] [13/Sep/2024:00:37:58] ENGINE Error in HTTPServer.serve 2024-09-13T00:37:59.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: Traceback (most recent call last): 2024-09-13T00:37:59.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:37:59.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: self._connections.run(self.expiration_interval) 2024-09-13T00:37:59.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:37:59.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: self._run(expiration_interval) 2024-09-13T00:37:59.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:37:59.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:37:59.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:37:59.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:37:59.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:37:59.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: s = self.context.wrap_socket( 2024-09-13T00:37:59.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:37:59.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: return self.sslsocket_class._create( 2024-09-13T00:37:59.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:37:59.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: self.do_handshake() 2024-09-13T00:37:59.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:37:59.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: self._sslobj.do_handshake() 2024-09-13T00:37:59.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:37:59.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: cephadm 2024-09-13T00:37:58.185253+0000 mgr.y (mgr.25064) 5 : cephadm [INF] [13/Sep/2024:00:37:58] ENGINE Serving on http://172.21.15.77:8765 2024-09-13T00:37:59.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: cephadm 2024-09-13T00:37:58.185340+0000 mgr.y (mgr.25064) 6 : cephadm [INF] [13/Sep/2024:00:37:58] ENGINE Bus STARTED 2024-09-13T00:37:59.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: cluster 2024-09-13T00:37:58.259261+0000 mgr.y (mgr.25064) 7 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:59.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: cluster 2024-09-13T00:37:58.266790+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2024-09-13T00:37:59.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: cluster 2024-09-13T00:37:58.267391+0000 mon.a (mon.0) 1149 : cluster [DBG] mgrmap e29: y(active, since 2s) 2024-09-13T00:37:59.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:37:59 smithi077 bash[32040]: audit 2024-09-13T00:37:58.863523+0000 mon.a (mon.0) 1150 : audit [DBG] from='client.? 172.21.15.77:0/3926927439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:59.576 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:37:59.578 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[53580]: debug 2024-09-13T00:37:59.294+0000 7fba2503e040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T00:37:59.578 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[53580]: debug 2024-09-13T00:37:59.370+0000 7fba2503e040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T00:37:59.579 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: cephadm 2024-09-13T00:37:57.964753+0000 mgr.y (mgr.25064) 2 : cephadm [INF] [13/Sep/2024:00:37:57] ENGINE Bus STARTING 2024-09-13T00:37:59.579 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: cephadm 2024-09-13T00:37:58.084168+0000 mgr.y (mgr.25064) 3 : cephadm [INF] [13/Sep/2024:00:37:58] ENGINE Serving on https://172.21.15.77:7150 2024-09-13T00:37:59.579 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: cephadm 2024-09-13T00:37:58.089166+0000 mgr.y (mgr.25064) 4 : cephadm [ERR] [13/Sep/2024:00:37:58] ENGINE Error in HTTPServer.serve 2024-09-13T00:37:59.579 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: Traceback (most recent call last): 2024-09-13T00:37:59.579 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:37:59.579 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: self._connections.run(self.expiration_interval) 2024-09-13T00:37:59.579 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:37:59.579 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: self._run(expiration_interval) 2024-09-13T00:37:59.579 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:37:59.580 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:37:59.580 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:37:59.580 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:37:59.580 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:37:59.580 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: s = self.context.wrap_socket( 2024-09-13T00:37:59.580 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:37:59.580 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: return self.sslsocket_class._create( 2024-09-13T00:37:59.580 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:37:59.580 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: self.do_handshake() 2024-09-13T00:37:59.580 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:37:59.580 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: self._sslobj.do_handshake() 2024-09-13T00:37:59.580 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:37:59.580 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: cephadm 2024-09-13T00:37:58.185253+0000 mgr.y (mgr.25064) 5 : cephadm [INF] [13/Sep/2024:00:37:58] ENGINE Serving on http://172.21.15.77:8765 2024-09-13T00:37:59.581 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: cephadm 2024-09-13T00:37:58.185340+0000 mgr.y (mgr.25064) 6 : cephadm [INF] [13/Sep/2024:00:37:58] ENGINE Bus STARTED 2024-09-13T00:37:59.581 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: cluster 2024-09-13T00:37:58.259261+0000 mgr.y (mgr.25064) 7 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:37:59.581 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: cluster 2024-09-13T00:37:58.266790+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e236: 8 total, 8 up, 8 in 2024-09-13T00:37:59.581 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: cluster 2024-09-13T00:37:58.267391+0000 mon.a (mon.0) 1149 : cluster [DBG] mgrmap e29: y(active, since 2s) 2024-09-13T00:37:59.581 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[24895]: audit 2024-09-13T00:37:58.863523+0000 mon.a (mon.0) 1150 : audit [DBG] from='client.? 172.21.15.77:0/3926927439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:37:59.594 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 41943040~10 2024-09-13T00:37:59.608 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:37:59.850 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[53580]: debug 2024-09-13T00:37:59.578+0000 7fba2503e040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T00:37:59.850 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[53580]: debug 2024-09-13T00:37:59.658+0000 7fba2503e040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T00:38:00.247 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:37:59 smithi088 bash[53580]: debug 2024-09-13T00:37:59.910+0000 7fba2503e040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T00:38:00.247 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:00 smithi088 bash[53580]: debug 2024-09-13T00:38:00.082+0000 7fba2503e040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:38:00.509 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:00 smithi088 bash[24895]: cluster 2024-09-13T00:37:59.266300+0000 mon.a (mon.0) 1151 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2024-09-13T00:38:00.509 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:00 smithi088 bash[53580]: debug 2024-09-13T00:38:00.246+0000 7fba2503e040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T00:38:00.509 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:00 smithi088 bash[53580]: debug 2024-09-13T00:38:00.330+0000 7fba2503e040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T00:38:00.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:00 smithi077 bash[22596]: cluster 2024-09-13T00:37:59.266300+0000 mon.a (mon.0) 1151 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2024-09-13T00:38:00.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:00 smithi077 bash[32040]: cluster 2024-09-13T00:37:59.266300+0000 mon.a (mon.0) 1151 : cluster [DBG] osdmap e237: 8 total, 8 up, 8 in 2024-09-13T00:38:00.850 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:00 smithi088 bash[53580]: debug 2024-09-13T00:38:00.506+0000 7fba2503e040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:38:01.315 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 41943040 2024-09-13T00:38:01.361 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:38:01.600 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:01 smithi088 bash[53580]: debug 2024-09-13T00:38:01.342+0000 7fba2503e040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T00:38:01.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:01 smithi088 bash[53580]: debug 2024-09-13T00:38:01.430+0000 7fba2503e040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T00:38:01.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:01 smithi088 bash[24895]: cluster 2024-09-13T00:38:00.260099+0000 mgr.y (mgr.25064) 8 : cluster [DBG] pgmap v8: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:01.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:01 smithi088 bash[24895]: cluster 2024-09-13T00:38:00.295188+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2024-09-13T00:38:01.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:01 smithi088 bash[24895]: cluster 2024-09-13T00:38:00.295901+0000 mon.a (mon.0) 1153 : cluster [DBG] mgrmap e30: y(active, since 4s) 2024-09-13T00:38:01.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:01 smithi088 bash[24895]: audit 2024-09-13T00:38:01.219244+0000 mon.c (mon.2) 200 : audit [DBG] from='client.? 172.21.15.77:0/1980495689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:01.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:01 smithi077 bash[22596]: cluster 2024-09-13T00:38:00.260099+0000 mgr.y (mgr.25064) 8 : cluster [DBG] pgmap v8: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:01.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:01 smithi077 bash[22596]: cluster 2024-09-13T00:38:00.295188+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2024-09-13T00:38:01.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:01 smithi077 bash[22596]: cluster 2024-09-13T00:38:00.295901+0000 mon.a (mon.0) 1153 : cluster [DBG] mgrmap e30: y(active, since 4s) 2024-09-13T00:38:01.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:01 smithi077 bash[22596]: audit 2024-09-13T00:38:01.219244+0000 mon.c (mon.2) 200 : audit [DBG] from='client.? 172.21.15.77:0/1980495689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:01.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:01 smithi077 bash[32040]: cluster 2024-09-13T00:38:00.260099+0000 mgr.y (mgr.25064) 8 : cluster [DBG] pgmap v8: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:01.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:01 smithi077 bash[32040]: cluster 2024-09-13T00:38:00.295188+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e238: 8 total, 8 up, 8 in 2024-09-13T00:38:01.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:01 smithi077 bash[32040]: cluster 2024-09-13T00:38:00.295901+0000 mon.a (mon.0) 1153 : cluster [DBG] mgrmap e30: y(active, since 4s) 2024-09-13T00:38:01.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:01 smithi077 bash[32040]: audit 2024-09-13T00:38:01.219244+0000 mon.c (mon.2) 200 : audit [DBG] from='client.? 172.21.15.77:0/1980495689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:02.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:01 smithi088 bash[53580]: debug 2024-09-13T00:38:01.662+0000 7fba2503e040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T00:38:02.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:01 smithi088 bash[53580]: debug 2024-09-13T00:38:01.734+0000 7fba2503e040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T00:38:02.441 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:38:02.456 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap3 2024-09-13T00:38:02.600 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:02 smithi088 bash[60003]: ts=2024-09-13T00:38:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:02.600 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:02 smithi088 bash[53580]: debug 2024-09-13T00:38:02.118+0000 7fba2503e040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T00:38:02.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:02 smithi088 bash[53580]: debug 2024-09-13T00:38:02.202+0000 7fba2503e040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T00:38:02.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:02 smithi088 bash[53580]: [13/Sep/2024:00:38:02] ENGINE Bus STARTING 2024-09-13T00:38:02.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:02 smithi088 bash[53580]: CherryPy Checker: 2024-09-13T00:38:02.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:02 smithi088 bash[53580]: The Application mounted at '' has an empty config. 2024-09-13T00:38:02.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:02 smithi088 bash[53580]: [13/Sep/2024:00:38:02] ENGINE Serving on http://:::9283 2024-09-13T00:38:02.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:02 smithi088 bash[53580]: [13/Sep/2024:00:38:02] ENGINE Bus STARTED 2024-09-13T00:38:02.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:02 smithi088 bash[24895]: cluster 2024-09-13T00:38:01.296220+0000 mon.a (mon.0) 1154 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2024-09-13T00:38:02.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:02 smithi088 bash[24895]: cluster 2024-09-13T00:38:02.211873+0000 mon.a (mon.0) 1155 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:38:02.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:02 smithi088 bash[24895]: audit 2024-09-13T00:38:02.213502+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.? 172.21.15.88:0/456778696' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:38:02.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:02 smithi088 bash[24895]: audit 2024-09-13T00:38:02.213975+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.? 172.21.15.88:0/456778696' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:38:02.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:02 smithi088 bash[24895]: audit 2024-09-13T00:38:02.216752+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.? 172.21.15.88:0/456778696' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:38:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:02 smithi088 bash[24895]: audit 2024-09-13T00:38:02.217206+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.? 172.21.15.88:0/456778696' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:38:02.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:02 smithi077 bash[32040]: cluster 2024-09-13T00:38:01.296220+0000 mon.a (mon.0) 1154 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2024-09-13T00:38:02.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:02 smithi077 bash[32040]: cluster 2024-09-13T00:38:02.211873+0000 mon.a (mon.0) 1155 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:38:02.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:02 smithi077 bash[32040]: audit 2024-09-13T00:38:02.213502+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.? 172.21.15.88:0/456778696' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:38:02.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:02 smithi077 bash[32040]: audit 2024-09-13T00:38:02.213975+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.? 172.21.15.88:0/456778696' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:38:02.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:02 smithi077 bash[32040]: audit 2024-09-13T00:38:02.216752+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.? 172.21.15.88:0/456778696' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:38:02.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:02 smithi077 bash[32040]: audit 2024-09-13T00:38:02.217206+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.? 172.21.15.88:0/456778696' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:38:02.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:02 smithi077 bash[22596]: cluster 2024-09-13T00:38:01.296220+0000 mon.a (mon.0) 1154 : cluster [DBG] osdmap e239: 8 total, 8 up, 8 in 2024-09-13T00:38:02.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:02 smithi077 bash[22596]: cluster 2024-09-13T00:38:02.211873+0000 mon.a (mon.0) 1155 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:38:02.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:02 smithi077 bash[22596]: audit 2024-09-13T00:38:02.213502+0000 mon.b (mon.1) 83 : audit [DBG] from='mgr.? 172.21.15.88:0/456778696' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:38:02.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:02 smithi077 bash[22596]: audit 2024-09-13T00:38:02.213975+0000 mon.b (mon.1) 84 : audit [DBG] from='mgr.? 172.21.15.88:0/456778696' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:38:02.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:02 smithi077 bash[22596]: audit 2024-09-13T00:38:02.216752+0000 mon.b (mon.1) 85 : audit [DBG] from='mgr.? 172.21.15.88:0/456778696' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:38:02.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:02 smithi077 bash[22596]: audit 2024-09-13T00:38:02.217206+0000 mon.b (mon.1) 86 : audit [DBG] from='mgr.? 172.21.15.88:0/456778696' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:38:03.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:03 smithi088 bash[24895]: cluster 2024-09-13T00:38:02.260993+0000 mgr.y (mgr.25064) 9 : cluster [DBG] pgmap v11: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 255 B/s wr, 0 op/s 2024-09-13T00:38:03.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:03 smithi088 bash[24895]: cluster 2024-09-13T00:38:02.298641+0000 mon.a (mon.0) 1156 : cluster [DBG] mgrmap e31: y(active, since 6s), standbys: x 2024-09-13T00:38:03.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:03 smithi088 bash[24895]: audit 2024-09-13T00:38:02.302349+0000 mon.c (mon.2) 201 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T00:38:03.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:03 smithi088 bash[24895]: cluster 2024-09-13T00:38:02.307051+0000 mon.a (mon.0) 1157 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2024-09-13T00:38:03.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:03 smithi088 bash[24895]: audit 2024-09-13T00:38:02.828787+0000 mon.a (mon.0) 1158 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:03.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:03 smithi088 bash[24895]: audit 2024-09-13T00:38:02.836538+0000 mon.a (mon.0) 1159 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:03.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:03 smithi088 bash[24895]: audit 2024-09-13T00:38:03.051533+0000 mon.a (mon.0) 1160 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:03.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:03 smithi088 bash[24895]: audit 2024-09-13T00:38:03.064132+0000 mon.a (mon.0) 1161 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:03.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:03 smithi088 bash[60003]: ts=2024-09-13T00:38:03.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:03.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:03 smithi077 bash[22596]: cluster 2024-09-13T00:38:02.260993+0000 mgr.y (mgr.25064) 9 : cluster [DBG] pgmap v11: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 255 B/s wr, 0 op/s 2024-09-13T00:38:03.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:03 smithi077 bash[22596]: cluster 2024-09-13T00:38:02.298641+0000 mon.a (mon.0) 1156 : cluster [DBG] mgrmap e31: y(active, since 6s), standbys: x 2024-09-13T00:38:03.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:03 smithi077 bash[22596]: audit 2024-09-13T00:38:02.302349+0000 mon.c (mon.2) 201 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T00:38:03.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:03 smithi077 bash[22596]: cluster 2024-09-13T00:38:02.307051+0000 mon.a (mon.0) 1157 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2024-09-13T00:38:03.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:03 smithi077 bash[22596]: audit 2024-09-13T00:38:02.828787+0000 mon.a (mon.0) 1158 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:03.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:03 smithi077 bash[22596]: audit 2024-09-13T00:38:02.836538+0000 mon.a (mon.0) 1159 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:03.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:03 smithi077 bash[22596]: audit 2024-09-13T00:38:03.051533+0000 mon.a (mon.0) 1160 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:03.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:03 smithi077 bash[22596]: audit 2024-09-13T00:38:03.064132+0000 mon.a (mon.0) 1161 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:03.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:03 smithi077 bash[32040]: cluster 2024-09-13T00:38:02.260993+0000 mgr.y (mgr.25064) 9 : cluster [DBG] pgmap v11: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s rd, 255 B/s wr, 0 op/s 2024-09-13T00:38:03.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:03 smithi077 bash[32040]: cluster 2024-09-13T00:38:02.298641+0000 mon.a (mon.0) 1156 : cluster [DBG] mgrmap e31: y(active, since 6s), standbys: x 2024-09-13T00:38:03.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:03 smithi077 bash[32040]: audit 2024-09-13T00:38:02.302349+0000 mon.c (mon.2) 201 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T00:38:03.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:03 smithi077 bash[32040]: cluster 2024-09-13T00:38:02.307051+0000 mon.a (mon.0) 1157 : cluster [DBG] osdmap e240: 8 total, 8 up, 8 in 2024-09-13T00:38:03.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:03 smithi077 bash[32040]: audit 2024-09-13T00:38:02.828787+0000 mon.a (mon.0) 1158 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:03.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:03 smithi077 bash[32040]: audit 2024-09-13T00:38:02.836538+0000 mon.a (mon.0) 1159 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:03.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:03 smithi077 bash[32040]: audit 2024-09-13T00:38:03.051533+0000 mon.a (mon.0) 1160 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:03.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:03 smithi077 bash[32040]: audit 2024-09-13T00:38:03.064132+0000 mon.a (mon.0) 1161 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:04.350 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 41943040~10 2024-09-13T00:38:04.365 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap4 2024-09-13T00:38:04.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:04 smithi088 bash[24895]: cluster 2024-09-13T00:38:03.314527+0000 mon.a (mon.0) 1162 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2024-09-13T00:38:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:04 smithi088 bash[24895]: audit 2024-09-13T00:38:03.537283+0000 mon.a (mon.0) 1163 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:04 smithi088 bash[24895]: audit 2024-09-13T00:38:03.545284+0000 mon.a (mon.0) 1164 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:04 smithi088 bash[24895]: audit 2024-09-13T00:38:03.575030+0000 mon.a (mon.0) 1165 : audit [DBG] from='client.? 172.21.15.77:0/305980984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:04 smithi088 bash[24895]: audit 2024-09-13T00:38:03.788544+0000 mon.a (mon.0) 1166 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:04 smithi088 bash[24895]: audit 2024-09-13T00:38:03.798957+0000 mon.a (mon.0) 1167 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:04.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:04 smithi077 bash[22596]: cluster 2024-09-13T00:38:03.314527+0000 mon.a (mon.0) 1162 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2024-09-13T00:38:04.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:04 smithi077 bash[22596]: audit 2024-09-13T00:38:03.537283+0000 mon.a (mon.0) 1163 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:04.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:04 smithi077 bash[22596]: audit 2024-09-13T00:38:03.545284+0000 mon.a (mon.0) 1164 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:04.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:04 smithi077 bash[22596]: audit 2024-09-13T00:38:03.575030+0000 mon.a (mon.0) 1165 : audit [DBG] from='client.? 172.21.15.77:0/305980984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:04.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:04 smithi077 bash[22596]: audit 2024-09-13T00:38:03.788544+0000 mon.a (mon.0) 1166 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:04.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:04 smithi077 bash[22596]: audit 2024-09-13T00:38:03.798957+0000 mon.a (mon.0) 1167 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:04.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:04 smithi077 bash[32040]: cluster 2024-09-13T00:38:03.314527+0000 mon.a (mon.0) 1162 : cluster [DBG] osdmap e241: 8 total, 8 up, 8 in 2024-09-13T00:38:04.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:04 smithi077 bash[32040]: audit 2024-09-13T00:38:03.537283+0000 mon.a (mon.0) 1163 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:04.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:04 smithi077 bash[32040]: audit 2024-09-13T00:38:03.545284+0000 mon.a (mon.0) 1164 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:04.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:04 smithi077 bash[32040]: audit 2024-09-13T00:38:03.575030+0000 mon.a (mon.0) 1165 : audit [DBG] from='client.? 172.21.15.77:0/305980984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:04.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:04 smithi077 bash[32040]: audit 2024-09-13T00:38:03.788544+0000 mon.a (mon.0) 1166 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:04.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:04 smithi077 bash[32040]: audit 2024-09-13T00:38:03.798957+0000 mon.a (mon.0) 1167 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:05.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:05 smithi088 bash[24895]: cluster 2024-09-13T00:38:04.261908+0000 mgr.y (mgr.25064) 10 : cluster [DBG] pgmap v14: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 261 KiB/s rd, 21 KiB/s wr, 317 op/s 2024-09-13T00:38:05.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:05 smithi088 bash[24895]: cluster 2024-09-13T00:38:04.330217+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2024-09-13T00:38:05.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:05 smithi077 bash[22596]: cluster 2024-09-13T00:38:04.261908+0000 mgr.y (mgr.25064) 10 : cluster [DBG] pgmap v14: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 261 KiB/s rd, 21 KiB/s wr, 317 op/s 2024-09-13T00:38:05.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:05 smithi077 bash[22596]: cluster 2024-09-13T00:38:04.330217+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2024-09-13T00:38:05.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:05 smithi077 bash[32040]: cluster 2024-09-13T00:38:04.261908+0000 mgr.y (mgr.25064) 10 : cluster [DBG] pgmap v14: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 261 KiB/s rd, 21 KiB/s wr, 317 op/s 2024-09-13T00:38:05.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:05 smithi077 bash[32040]: cluster 2024-09-13T00:38:04.330217+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e242: 8 total, 8 up, 8 in 2024-09-13T00:38:06.202 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 41943040 2024-09-13T00:38:06.249 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:38:06.383 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:38:06.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:06 smithi088 bash[24895]: cluster 2024-09-13T00:38:05.343046+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2024-09-13T00:38:06.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:06 smithi088 bash[24895]: audit 2024-09-13T00:38:05.939777+0000 mon.c (mon.2) 202 : audit [DBG] from='client.? 172.21.15.77:0/4069461060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:06.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:06 smithi088 bash[24895]: cluster 2024-09-13T00:38:06.174235+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2024-09-13T00:38:06.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:06 smithi077 bash[22596]: cluster 2024-09-13T00:38:05.343046+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2024-09-13T00:38:06.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:06 smithi077 bash[22596]: audit 2024-09-13T00:38:05.939777+0000 mon.c (mon.2) 202 : audit [DBG] from='client.? 172.21.15.77:0/4069461060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:06.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:06 smithi077 bash[22596]: cluster 2024-09-13T00:38:06.174235+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2024-09-13T00:38:06.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:06 smithi077 bash[32040]: cluster 2024-09-13T00:38:05.343046+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e243: 8 total, 8 up, 8 in 2024-09-13T00:38:06.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:06 smithi077 bash[32040]: audit 2024-09-13T00:38:05.939777+0000 mon.c (mon.2) 202 : audit [DBG] from='client.? 172.21.15.77:0/4069461060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:06.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:06 smithi077 bash[32040]: cluster 2024-09-13T00:38:06.174235+0000 mon.a (mon.0) 1170 : cluster [DBG] osdmap e244: 8 total, 8 up, 8 in 2024-09-13T00:38:07.649 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.SnapTrimBeforePrepare (8153 ms) 2024-09-13T00:38:07.650 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.AbortInUseImage 2024-09-13T00:38:07.683 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:07 smithi088 bash[24895]: cluster 2024-09-13T00:38:06.262785+0000 mgr.y (mgr.25064) 11 : cluster [DBG] pgmap v18: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:07.733 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:38:07.760 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:38:07.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:07 smithi077 bash[22596]: cluster 2024-09-13T00:38:06.262785+0000 mgr.y (mgr.25064) 11 : cluster [DBG] pgmap v18: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:07.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:07 smithi077 bash[32040]: cluster 2024-09-13T00:38:06.262785+0000 mgr.y (mgr.25064) 11 : cluster [DBG] pgmap v18: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:08.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:07 smithi088 bash[53580]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:38:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:38:08.111 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.AbortInUseImage (462 ms) 2024-09-13T00:38:08.112 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.AbortWithoutSnapshots 2024-09-13T00:38:08.171 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:38:08.196 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:38:08.277 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 16777218~8388609 2024-09-13T00:38:08.353 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:38:08.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:08 smithi077 bash[22596]: cluster 2024-09-13T00:38:07.354922+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2024-09-13T00:38:08.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:08 smithi077 bash[22596]: audit 2024-09-13T00:38:08.334559+0000 mon.a (mon.0) 1172 : audit [DBG] from='client.? 172.21.15.77:0/438368744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:08.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:08 smithi077 bash[32040]: cluster 2024-09-13T00:38:07.354922+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2024-09-13T00:38:08.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:08 smithi077 bash[32040]: audit 2024-09-13T00:38:08.334559+0000 mon.a (mon.0) 1172 : audit [DBG] from='client.? 172.21.15.77:0/438368744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:08.782 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:38:08.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:08 smithi088 bash[24895]: cluster 2024-09-13T00:38:07.354922+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e245: 8 total, 8 up, 8 in 2024-09-13T00:38:08.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:08 smithi088 bash[24895]: audit 2024-09-13T00:38:08.334559+0000 mon.a (mon.0) 1172 : audit [DBG] from='client.? 172.21.15.77:0/438368744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:08.855 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 16777218~8388609 2024-09-13T00:38:08.962 INFO:tasks.workunit.client.0.smithi077.stdout:migration_abort 2024-09-13T00:38:09.330 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.AbortWithoutSnapshots (1218 ms) 2024-09-13T00:38:09.330 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.AbortWithSnapshots 2024-09-13T00:38:09.409 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:38:09.439 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:38:09.545 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:38:09.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:09 smithi077 bash[32040]: cluster 2024-09-13T00:38:08.263594+0000 mgr.y (mgr.25064) 12 : cluster [DBG] pgmap v20: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 213 KiB/s rd, 16 KiB/s wr, 273 op/s 2024-09-13T00:38:09.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:09 smithi077 bash[22596]: cluster 2024-09-13T00:38:08.263594+0000 mgr.y (mgr.25064) 12 : cluster [DBG] pgmap v20: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 213 KiB/s rd, 16 KiB/s wr, 273 op/s 2024-09-13T00:38:09.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:09 smithi088 bash[24895]: cluster 2024-09-13T00:38:08.263594+0000 mgr.y (mgr.25064) 12 : cluster [DBG] pgmap v20: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 213 KiB/s rd, 16 KiB/s wr, 273 op/s 2024-09-13T00:38:11.421 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 8388609~8388609 2024-09-13T00:38:11.460 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 0~1000 2024-09-13T00:38:11.460 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 1010~1000 2024-09-13T00:38:11.460 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap2 2024-09-13T00:38:11.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:11 smithi077 bash[22596]: cluster 2024-09-13T00:38:10.264379+0000 mgr.y (mgr.25064) 13 : cluster [DBG] pgmap v21: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 143 KiB/s rd, 11 KiB/s wr, 184 op/s 2024-09-13T00:38:11.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:11 smithi077 bash[22596]: cluster 2024-09-13T00:38:10.376970+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2024-09-13T00:38:11.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:11 smithi077 bash[22596]: audit 2024-09-13T00:38:10.699879+0000 mon.a (mon.0) 1174 : audit [DBG] from='client.? 172.21.15.77:0/3897752206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:11.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:11 smithi077 bash[32040]: cluster 2024-09-13T00:38:10.264379+0000 mgr.y (mgr.25064) 13 : cluster [DBG] pgmap v21: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 143 KiB/s rd, 11 KiB/s wr, 184 op/s 2024-09-13T00:38:11.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:11 smithi077 bash[32040]: cluster 2024-09-13T00:38:10.376970+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2024-09-13T00:38:11.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:11 smithi077 bash[32040]: audit 2024-09-13T00:38:10.699879+0000 mon.a (mon.0) 1174 : audit [DBG] from='client.? 172.21.15.77:0/3897752206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:11.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:11 smithi088 bash[24895]: cluster 2024-09-13T00:38:10.264379+0000 mgr.y (mgr.25064) 13 : cluster [DBG] pgmap v21: 137 pgs: 137 active+clean; 781 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 143 KiB/s rd, 11 KiB/s wr, 184 op/s 2024-09-13T00:38:11.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:11 smithi088 bash[24895]: cluster 2024-09-13T00:38:10.376970+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e246: 8 total, 8 up, 8 in 2024-09-13T00:38:11.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:11 smithi088 bash[24895]: audit 2024-09-13T00:38:10.699879+0000 mon.a (mon.0) 1174 : audit [DBG] from='client.? 172.21.15.77:0/3897752206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:12.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:12 smithi077 bash[22596]: cluster 2024-09-13T00:38:11.396966+0000 mon.a (mon.0) 1175 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2024-09-13T00:38:12.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:12 smithi077 bash[22596]: audit 2024-09-13T00:38:11.847331+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:12.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:12 smithi077 bash[32040]: cluster 2024-09-13T00:38:11.396966+0000 mon.a (mon.0) 1175 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2024-09-13T00:38:12.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:12 smithi077 bash[32040]: audit 2024-09-13T00:38:11.847331+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:12.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:12 smithi088 bash[24895]: cluster 2024-09-13T00:38:11.396966+0000 mon.a (mon.0) 1175 : cluster [DBG] osdmap e247: 8 total, 8 up, 8 in 2024-09-13T00:38:12.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:12 smithi088 bash[24895]: audit 2024-09-13T00:38:11.847331+0000 mon.c (mon.2) 203 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:12.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:12 smithi088 bash[60003]: ts=2024-09-13T00:38:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:13.437 INFO:tasks.workunit.client.0.smithi077.stdout:write: X 8388609~1000 2024-09-13T00:38:13.455 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 16777228~1000 2024-09-13T00:38:13.455 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 167772160 2024-09-13T00:38:13.476 INFO:tasks.workunit.client.0.smithi077.stdout:write: 2 83886079~8388609 2024-09-13T00:38:13.526 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap3 2024-09-13T00:38:13.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:13 smithi077 bash[22596]: cluster 2024-09-13T00:38:12.265237+0000 mgr.y (mgr.25064) 14 : cluster [DBG] pgmap v24: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 175 KiB/s rd, 12 KiB/s wr, 223 op/s 2024-09-13T00:38:13.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:13 smithi077 bash[22596]: cluster 2024-09-13T00:38:12.408170+0000 mon.a (mon.0) 1176 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2024-09-13T00:38:13.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:13 smithi077 bash[22596]: audit 2024-09-13T00:38:13.057780+0000 mon.c (mon.2) 204 : audit [DBG] from='client.? 172.21.15.77:0/2658970780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:13.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:13 smithi077 bash[32040]: cluster 2024-09-13T00:38:12.265237+0000 mgr.y (mgr.25064) 14 : cluster [DBG] pgmap v24: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 175 KiB/s rd, 12 KiB/s wr, 223 op/s 2024-09-13T00:38:13.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:13 smithi077 bash[32040]: cluster 2024-09-13T00:38:12.408170+0000 mon.a (mon.0) 1176 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2024-09-13T00:38:13.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:13 smithi077 bash[32040]: audit 2024-09-13T00:38:13.057780+0000 mon.c (mon.2) 204 : audit [DBG] from='client.? 172.21.15.77:0/2658970780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:13.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:13 smithi088 bash[24895]: cluster 2024-09-13T00:38:12.265237+0000 mgr.y (mgr.25064) 14 : cluster [DBG] pgmap v24: 137 pgs: 137 active+clean; 781 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 175 KiB/s rd, 12 KiB/s wr, 223 op/s 2024-09-13T00:38:13.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:13 smithi088 bash[24895]: cluster 2024-09-13T00:38:12.408170+0000 mon.a (mon.0) 1176 : cluster [DBG] osdmap e248: 8 total, 8 up, 8 in 2024-09-13T00:38:13.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:13 smithi088 bash[24895]: audit 2024-09-13T00:38:13.057780+0000 mon.c (mon.2) 204 : audit [DBG] from='client.? 172.21.15.77:0/2658970780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:13.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:13 smithi088 bash[60003]: ts=2024-09-13T00:38:13.510Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:14.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:14 smithi077 bash[22596]: cluster 2024-09-13T00:38:13.408492+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2024-09-13T00:38:14.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:14 smithi077 bash[22596]: audit 2024-09-13T00:38:14.071748+0000 mon.a (mon.0) 1178 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:14.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:14 smithi077 bash[22596]: audit 2024-09-13T00:38:14.085068+0000 mon.a (mon.0) 1179 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:14.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:14 smithi077 bash[22596]: audit 2024-09-13T00:38:14.089999+0000 mon.c (mon.2) 205 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:38:14.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:14 smithi077 bash[22596]: audit 2024-09-13T00:38:14.091797+0000 mon.c (mon.2) 206 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:38:14.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:14 smithi077 bash[32040]: cluster 2024-09-13T00:38:13.408492+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2024-09-13T00:38:14.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:14 smithi077 bash[32040]: audit 2024-09-13T00:38:14.071748+0000 mon.a (mon.0) 1178 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:14.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:14 smithi077 bash[32040]: audit 2024-09-13T00:38:14.085068+0000 mon.a (mon.0) 1179 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:14.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:14 smithi077 bash[32040]: audit 2024-09-13T00:38:14.089999+0000 mon.c (mon.2) 205 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:38:14.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:14 smithi077 bash[32040]: audit 2024-09-13T00:38:14.091797+0000 mon.c (mon.2) 206 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:38:14.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:14 smithi088 bash[24895]: cluster 2024-09-13T00:38:13.408492+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e249: 8 total, 8 up, 8 in 2024-09-13T00:38:14.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:14 smithi088 bash[24895]: audit 2024-09-13T00:38:14.071748+0000 mon.a (mon.0) 1178 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:14.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:14 smithi088 bash[24895]: audit 2024-09-13T00:38:14.085068+0000 mon.a (mon.0) 1179 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:14.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:14 smithi088 bash[24895]: audit 2024-09-13T00:38:14.089999+0000 mon.c (mon.2) 205 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:38:14.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:14 smithi088 bash[24895]: audit 2024-09-13T00:38:14.091797+0000 mon.c (mon.2) 206 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:38:15.448 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:38:15.492 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 83886079~1 2024-09-13T00:38:15.511 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:38:15.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[32040]: cephadm 2024-09-13T00:38:14.094148+0000 mgr.y (mgr.25064) 15 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.conf 2024-09-13T00:38:15.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[32040]: cephadm 2024-09-13T00:38:14.094801+0000 mgr.y (mgr.25064) 16 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:38:15.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[32040]: cluster 2024-09-13T00:38:14.265888+0000 mgr.y (mgr.25064) 17 : cluster [DBG] pgmap v27: 137 pgs: 137 active+clean; 845 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 29 MiB/s wr, 488 op/s 2024-09-13T00:38:15.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[32040]: cephadm 2024-09-13T00:38:14.283740+0000 mgr.y (mgr.25064) 18 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:38:15.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[32040]: cephadm 2024-09-13T00:38:14.287589+0000 mgr.y (mgr.25064) 19 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:38:15.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[32040]: cephadm 2024-09-13T00:38:14.407648+0000 mgr.y (mgr.25064) 20 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:38:15.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[32040]: cluster 2024-09-13T00:38:14.422825+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2024-09-13T00:38:15.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[32040]: cephadm 2024-09-13T00:38:14.433098+0000 mgr.y (mgr.25064) 21 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:38:15.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[32040]: audit 2024-09-13T00:38:14.674274+0000 mon.a (mon.0) 1181 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:15.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[32040]: audit 2024-09-13T00:38:14.684728+0000 mon.a (mon.0) 1182 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:15.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[32040]: audit 2024-09-13T00:38:14.703965+0000 mon.a (mon.0) 1183 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:15.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[32040]: audit 2024-09-13T00:38:14.714228+0000 mon.a (mon.0) 1184 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:15.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[32040]: audit 2024-09-13T00:38:14.723762+0000 mon.a (mon.0) 1185 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:15.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[32040]: audit 2024-09-13T00:38:15.418341+0000 mon.a (mon.0) 1186 : audit [DBG] from='client.? 172.21.15.77:0/2557527176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:15.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[22596]: cephadm 2024-09-13T00:38:14.094148+0000 mgr.y (mgr.25064) 15 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.conf 2024-09-13T00:38:15.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[22596]: cephadm 2024-09-13T00:38:14.094801+0000 mgr.y (mgr.25064) 16 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:38:15.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[22596]: cluster 2024-09-13T00:38:14.265888+0000 mgr.y (mgr.25064) 17 : cluster [DBG] pgmap v27: 137 pgs: 137 active+clean; 845 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 29 MiB/s wr, 488 op/s 2024-09-13T00:38:15.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[22596]: cephadm 2024-09-13T00:38:14.283740+0000 mgr.y (mgr.25064) 18 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:38:15.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[22596]: cephadm 2024-09-13T00:38:14.287589+0000 mgr.y (mgr.25064) 19 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:38:15.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[22596]: cephadm 2024-09-13T00:38:14.407648+0000 mgr.y (mgr.25064) 20 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:38:15.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[22596]: cluster 2024-09-13T00:38:14.422825+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2024-09-13T00:38:15.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[22596]: cephadm 2024-09-13T00:38:14.433098+0000 mgr.y (mgr.25064) 21 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:38:15.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[22596]: audit 2024-09-13T00:38:14.674274+0000 mon.a (mon.0) 1181 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:15.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[22596]: audit 2024-09-13T00:38:14.684728+0000 mon.a (mon.0) 1182 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:15.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[22596]: audit 2024-09-13T00:38:14.703965+0000 mon.a (mon.0) 1183 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:15.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[22596]: audit 2024-09-13T00:38:14.714228+0000 mon.a (mon.0) 1184 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:15.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[22596]: audit 2024-09-13T00:38:14.723762+0000 mon.a (mon.0) 1185 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:15.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:15 smithi077 bash[22596]: audit 2024-09-13T00:38:15.418341+0000 mon.a (mon.0) 1186 : audit [DBG] from='client.? 172.21.15.77:0/2557527176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:15.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:15 smithi088 bash[24895]: cephadm 2024-09-13T00:38:14.094148+0000 mgr.y (mgr.25064) 15 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.conf 2024-09-13T00:38:15.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:15 smithi088 bash[24895]: cephadm 2024-09-13T00:38:14.094801+0000 mgr.y (mgr.25064) 16 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:38:15.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:15 smithi088 bash[24895]: cluster 2024-09-13T00:38:14.265888+0000 mgr.y (mgr.25064) 17 : cluster [DBG] pgmap v27: 137 pgs: 137 active+clean; 845 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 21 MiB/s rd, 29 MiB/s wr, 488 op/s 2024-09-13T00:38:15.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:15 smithi088 bash[24895]: cephadm 2024-09-13T00:38:14.283740+0000 mgr.y (mgr.25064) 18 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:38:15.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:15 smithi088 bash[24895]: cephadm 2024-09-13T00:38:14.287589+0000 mgr.y (mgr.25064) 19 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:38:15.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:15 smithi088 bash[24895]: cephadm 2024-09-13T00:38:14.407648+0000 mgr.y (mgr.25064) 20 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:38:15.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:15 smithi088 bash[24895]: cluster 2024-09-13T00:38:14.422825+0000 mon.a (mon.0) 1180 : cluster [DBG] osdmap e250: 8 total, 8 up, 8 in 2024-09-13T00:38:15.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:15 smithi088 bash[24895]: cephadm 2024-09-13T00:38:14.433098+0000 mgr.y (mgr.25064) 21 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:38:15.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:15 smithi088 bash[24895]: audit 2024-09-13T00:38:14.674274+0000 mon.a (mon.0) 1181 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:15.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:15 smithi088 bash[24895]: audit 2024-09-13T00:38:14.684728+0000 mon.a (mon.0) 1182 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:15.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:15 smithi088 bash[24895]: audit 2024-09-13T00:38:14.703965+0000 mon.a (mon.0) 1183 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:15.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:15 smithi088 bash[24895]: audit 2024-09-13T00:38:14.714228+0000 mon.a (mon.0) 1184 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:15.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:15 smithi088 bash[24895]: audit 2024-09-13T00:38:14.723762+0000 mon.a (mon.0) 1185 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:15.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:15 smithi088 bash[24895]: audit 2024-09-13T00:38:15.418341+0000 mon.a (mon.0) 1186 : audit [DBG] from='client.? 172.21.15.77:0/2557527176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:16.709 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:16 smithi088 bash[24895]: cephadm 2024-09-13T00:38:14.537731+0000 mgr.y (mgr.25064) 22 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:38:16.709 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:16 smithi088 bash[24895]: cephadm 2024-09-13T00:38:14.566601+0000 mgr.y (mgr.25064) 23 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:38:16.709 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:16 smithi088 bash[24895]: cephadm 2024-09-13T00:38:14.761789+0000 mgr.y (mgr.25064) 24 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T00:38:16.709 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:16 smithi088 bash[24895]: cephadm 2024-09-13T00:38:15.018819+0000 mgr.y (mgr.25064) 25 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi088 2024-09-13T00:38:16.709 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:16 smithi088 bash[24895]: cluster 2024-09-13T00:38:15.426425+0000 mon.a (mon.0) 1187 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2024-09-13T00:38:16.709 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:16 smithi088 bash[24895]: cluster 2024-09-13T00:38:16.179180+0000 mon.a (mon.0) 1188 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2024-09-13T00:38:16.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:16 smithi077 bash[22596]: cephadm 2024-09-13T00:38:14.537731+0000 mgr.y (mgr.25064) 22 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:38:16.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:16 smithi077 bash[22596]: cephadm 2024-09-13T00:38:14.566601+0000 mgr.y (mgr.25064) 23 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:38:16.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:16 smithi077 bash[22596]: cephadm 2024-09-13T00:38:14.761789+0000 mgr.y (mgr.25064) 24 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T00:38:16.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:16 smithi077 bash[22596]: cephadm 2024-09-13T00:38:15.018819+0000 mgr.y (mgr.25064) 25 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi088 2024-09-13T00:38:16.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:16 smithi077 bash[22596]: cluster 2024-09-13T00:38:15.426425+0000 mon.a (mon.0) 1187 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2024-09-13T00:38:16.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:16 smithi077 bash[22596]: cluster 2024-09-13T00:38:16.179180+0000 mon.a (mon.0) 1188 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2024-09-13T00:38:16.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:16 smithi077 bash[32040]: cephadm 2024-09-13T00:38:14.537731+0000 mgr.y (mgr.25064) 22 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:38:16.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:16 smithi077 bash[32040]: cephadm 2024-09-13T00:38:14.566601+0000 mgr.y (mgr.25064) 23 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:38:16.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:16 smithi077 bash[32040]: cephadm 2024-09-13T00:38:14.761789+0000 mgr.y (mgr.25064) 24 : cephadm [INF] Reconfiguring prometheus.a (dependencies changed)... 2024-09-13T00:38:16.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:16 smithi077 bash[32040]: cephadm 2024-09-13T00:38:15.018819+0000 mgr.y (mgr.25064) 25 : cephadm [INF] Reconfiguring daemon prometheus.a on smithi088 2024-09-13T00:38:16.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:16 smithi077 bash[32040]: cluster 2024-09-13T00:38:15.426425+0000 mon.a (mon.0) 1187 : cluster [DBG] osdmap e251: 8 total, 8 up, 8 in 2024-09-13T00:38:16.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:16 smithi077 bash[32040]: cluster 2024-09-13T00:38:16.179180+0000 mon.a (mon.0) 1188 : cluster [DBG] osdmap e252: 8 total, 8 up, 8 in 2024-09-13T00:38:17.682 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:17 smithi088 bash[24895]: cluster 2024-09-13T00:38:16.266626+0000 mgr.y (mgr.25064) 26 : cluster [DBG] pgmap v31: 137 pgs: 137 active+clean; 845 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:17.682 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:17 smithi088 bash[24895]: cluster 2024-09-13T00:38:17.187055+0000 mon.a (mon.0) 1189 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2024-09-13T00:38:17.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:17 smithi077 bash[22596]: cluster 2024-09-13T00:38:16.266626+0000 mgr.y (mgr.25064) 26 : cluster [DBG] pgmap v31: 137 pgs: 137 active+clean; 845 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:17.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:17 smithi077 bash[22596]: cluster 2024-09-13T00:38:17.187055+0000 mon.a (mon.0) 1189 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2024-09-13T00:38:17.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:17 smithi077 bash[32040]: cluster 2024-09-13T00:38:16.266626+0000 mgr.y (mgr.25064) 26 : cluster [DBG] pgmap v31: 137 pgs: 137 active+clean; 845 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:38:17.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:17 smithi077 bash[32040]: cluster 2024-09-13T00:38:17.187055+0000 mon.a (mon.0) 1189 : cluster [DBG] osdmap e253: 8 total, 8 up, 8 in 2024-09-13T00:38:18.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:17 smithi088 bash[53580]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:38:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-13T00:38:18.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:18 smithi077 bash[32040]: audit 2024-09-13T00:38:17.774096+0000 mon.c (mon.2) 207 : audit [DBG] from='client.? 172.21.15.77:0/2616954159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:18.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:18 smithi077 bash[32040]: cluster 2024-09-13T00:38:18.190350+0000 mon.a (mon.0) 1190 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2024-09-13T00:38:18.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:18 smithi077 bash[22596]: audit 2024-09-13T00:38:17.774096+0000 mon.c (mon.2) 207 : audit [DBG] from='client.? 172.21.15.77:0/2616954159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:18.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:18 smithi077 bash[22596]: cluster 2024-09-13T00:38:18.190350+0000 mon.a (mon.0) 1190 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2024-09-13T00:38:18.824 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:18 smithi088 bash[24895]: audit 2024-09-13T00:38:17.774096+0000 mon.c (mon.2) 207 : audit [DBG] from='client.? 172.21.15.77:0/2616954159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:18.824 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:18 smithi088 bash[24895]: cluster 2024-09-13T00:38:18.190350+0000 mon.a (mon.0) 1190 : cluster [DBG] osdmap e254: 8 total, 8 up, 8 in 2024-09-13T00:38:19.092 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:38:19.231 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 16777218~8388609 2024-09-13T00:38:19.422 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 systemd[1]: Stopping Ceph prometheus.a for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:38:19.423 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 bash[60003]: ts=2024-09-13T00:38:19.255Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T00:38:19.423 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 bash[60003]: ts=2024-09-13T00:38:19.256Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T00:38:19.423 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 bash[60003]: ts=2024-09-13T00:38:19.256Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T00:38:19.423 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 bash[60003]: ts=2024-09-13T00:38:19.256Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T00:38:19.423 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 bash[60003]: ts=2024-09-13T00:38:19.256Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T00:38:19.423 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 bash[60003]: ts=2024-09-13T00:38:19.256Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T00:38:19.423 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 bash[60003]: ts=2024-09-13T00:38:19.256Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T00:38:19.423 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 bash[60003]: ts=2024-09-13T00:38:19.256Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T00:38:19.423 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 bash[60003]: ts=2024-09-13T00:38:19.256Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T00:38:19.423 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 bash[60003]: ts=2024-09-13T00:38:19.256Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T00:38:19.423 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 bash[60003]: ts=2024-09-13T00:38:19.256Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T00:38:19.423 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 bash[60003]: ts=2024-09-13T00:38:19.256Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T00:38:19.679 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:19 smithi088 bash[24895]: cluster 2024-09-13T00:38:18.267351+0000 mgr.y (mgr.25064) 27 : cluster [DBG] pgmap v34: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 126 KiB/s rd, 4.2 MiB/s wr, 180 op/s 2024-09-13T00:38:19.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:19 smithi077 bash[22596]: cluster 2024-09-13T00:38:18.267351+0000 mgr.y (mgr.25064) 27 : cluster [DBG] pgmap v34: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 126 KiB/s rd, 4.2 MiB/s wr, 180 op/s 2024-09-13T00:38:19.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:19 smithi077 bash[32040]: cluster 2024-09-13T00:38:18.267351+0000 mgr.y (mgr.25064) 27 : cluster [DBG] pgmap v34: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 126 KiB/s rd, 4.2 MiB/s wr, 180 op/s 2024-09-13T00:38:19.987 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 bash[61702]: ceph-9218306c-7165-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T00:38:19.987 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 bash[61764]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-prometheus-a 2024-09-13T00:38:19.988 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-13T00:38:19.988 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 systemd[1]: Stopped Ceph prometheus.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:38:19.988 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:19 smithi088 systemd[1]: Started Ceph prometheus.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:38:20.213 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 0~8388609 2024-09-13T00:38:20.246 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 16777218~8388609 2024-09-13T00:38:20.357 INFO:tasks.workunit.client.0.smithi077.stdout:migration_abort 2024-09-13T00:38:21.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:20 smithi077 bash[22596]: audit 2024-09-13T00:38:19.919595+0000 mon.a (mon.0) 1191 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:21.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:20 smithi077 bash[22596]: audit 2024-09-13T00:38:19.931647+0000 mon.a (mon.0) 1192 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:21.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:20 smithi077 bash[22596]: audit 2024-09-13T00:38:19.938162+0000 mon.c (mon.2) 208 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:38:21.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:20 smithi077 bash[22596]: audit 2024-09-13T00:38:19.938763+0000 mgr.y (mgr.25064) 28 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:38:21.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:20 smithi077 bash[22596]: audit 2024-09-13T00:38:19.979986+0000 mon.c (mon.2) 209 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:38:21.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:20 smithi077 bash[22596]: audit 2024-09-13T00:38:20.139258+0000 mon.c (mon.2) 210 : audit [DBG] from='client.? 172.21.15.77:0/1105330818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:21.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:20 smithi077 bash[22596]: cluster 2024-09-13T00:38:20.190958+0000 mon.a (mon.0) 1193 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2024-09-13T00:38:21.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:20 smithi077 bash[22596]: cluster 2024-09-13T00:38:20.268236+0000 mgr.y (mgr.25064) 29 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 118 KiB/s rd, 3.9 MiB/s wr, 169 op/s 2024-09-13T00:38:21.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:20 smithi077 bash[32040]: audit 2024-09-13T00:38:19.919595+0000 mon.a (mon.0) 1191 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:21.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:20 smithi077 bash[32040]: audit 2024-09-13T00:38:19.931647+0000 mon.a (mon.0) 1192 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:21.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:20 smithi077 bash[32040]: audit 2024-09-13T00:38:19.938162+0000 mon.c (mon.2) 208 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:38:21.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:20 smithi077 bash[32040]: audit 2024-09-13T00:38:19.938763+0000 mgr.y (mgr.25064) 28 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:38:21.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:20 smithi077 bash[32040]: audit 2024-09-13T00:38:19.979986+0000 mon.c (mon.2) 209 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:38:21.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:20 smithi077 bash[32040]: audit 2024-09-13T00:38:20.139258+0000 mon.c (mon.2) 210 : audit [DBG] from='client.? 172.21.15.77:0/1105330818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:21.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:20 smithi077 bash[32040]: cluster 2024-09-13T00:38:20.190958+0000 mon.a (mon.0) 1193 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2024-09-13T00:38:21.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:20 smithi077 bash[32040]: cluster 2024-09-13T00:38:20.268236+0000 mgr.y (mgr.25064) 29 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 118 KiB/s rd, 3.9 MiB/s wr, 169 op/s 2024-09-13T00:38:21.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:20 smithi088 bash[24895]: audit 2024-09-13T00:38:19.919595+0000 mon.a (mon.0) 1191 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:21.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:20 smithi088 bash[24895]: audit 2024-09-13T00:38:19.931647+0000 mon.a (mon.0) 1192 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:21.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:20 smithi088 bash[24895]: audit 2024-09-13T00:38:19.938162+0000 mon.c (mon.2) 208 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:38:21.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:20 smithi088 bash[24895]: audit 2024-09-13T00:38:19.938763+0000 mgr.y (mgr.25064) 28 : audit [DBG] from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-13T00:38:21.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:20 smithi088 bash[24895]: audit 2024-09-13T00:38:19.979986+0000 mon.c (mon.2) 209 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:38:21.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:20 smithi088 bash[24895]: audit 2024-09-13T00:38:20.139258+0000 mon.c (mon.2) 210 : audit [DBG] from='client.? 172.21.15.77:0/1105330818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:21.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:20 smithi088 bash[24895]: cluster 2024-09-13T00:38:20.190958+0000 mon.a (mon.0) 1193 : cluster [DBG] osdmap e255: 8 total, 8 up, 8 in 2024-09-13T00:38:21.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:20 smithi088 bash[24895]: cluster 2024-09-13T00:38:20.268236+0000 mgr.y (mgr.25064) 29 : cluster [DBG] pgmap v36: 137 pgs: 137 active+clean; 869 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 118 KiB/s rd, 3.9 MiB/s wr, 169 op/s 2024-09-13T00:38:21.840 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.610Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-13T00:38:21.840 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.610Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-13T00:38:21.840 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.610Z caller=main.go:570 level=info host_details="(Linux 5.15.0-121-generic #131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024 x86_64 smithi088 (none))" 2024-09-13T00:38:21.840 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.610Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-13T00:38:21.840 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.610Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-13T00:38:21.840 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.612Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-13T00:38:21.840 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.612Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-13T00:38:21.840 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.613Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-13T00:38:21.841 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.613Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-13T00:38:21.841 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.617Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-13T00:38:21.841 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.617Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.18µs 2024-09-13T00:38:21.841 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.617Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-13T00:38:21.841 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.617Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-13T00:38:21.841 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.639Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-13T00:38:21.842 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.650Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-13T00:38:21.842 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.650Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-13T00:38:21.842 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.650Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=33.281µs wal_replay_duration=33.594875ms wbl_replay_duration=136ns total_replay_duration=33.649017ms 2024-09-13T00:38:21.842 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.654Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-13T00:38:21.842 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.654Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-13T00:38:21.843 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.654Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-13T00:38:21.843 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.676Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.972824ms db_storage=1.068µs remote_storage=1.262µs web_handler=450ns query_engine=843ns scrape=250.27µs scrape_sd=102.067µs notify=11.831µs notify_sd=10.162µs rules=21.155502ms tracing=6.557µs 2024-09-13T00:38:21.843 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.676Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-13T00:38:21.843 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:21 smithi088 bash[61791]: ts=2024-09-13T00:38:21.676Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-13T00:38:22.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:22 smithi077 bash[22596]: cluster 2024-09-13T00:38:21.203793+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2024-09-13T00:38:22.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:22 smithi077 bash[32040]: cluster 2024-09-13T00:38:21.203793+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2024-09-13T00:38:22.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:22 smithi088 bash[24895]: cluster 2024-09-13T00:38:21.203793+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e256: 8 total, 8 up, 8 in 2024-09-13T00:38:22.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:22 smithi088 bash[61791]: ts=2024-09-13T00:38:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:23.505 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:23 smithi088 bash[24895]: cluster 2024-09-13T00:38:22.217013+0000 mon.a (mon.0) 1195 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2024-09-13T00:38:23.505 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:23 smithi088 bash[24895]: cluster 2024-09-13T00:38:22.268812+0000 mgr.y (mgr.25064) 30 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 873 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s rd, 1004 KiB/s wr, 2 op/s 2024-09-13T00:38:23.505 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:23 smithi088 bash[24895]: cephadm 2024-09-13T00:38:22.480621+0000 mgr.y (mgr.25064) 31 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-13T00:38:23.505 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:23 smithi088 bash[24895]: audit 2024-09-13T00:38:22.481221+0000 mon.c (mon.2) 211 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:38:23.505 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:23 smithi088 bash[24895]: audit 2024-09-13T00:38:22.481986+0000 mon.a (mon.0) 1196 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:38:23.506 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:23 smithi088 bash[24895]: audit 2024-09-13T00:38:22.483767+0000 mon.c (mon.2) 212 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:38:23.506 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:23 smithi088 bash[24895]: audit 2024-09-13T00:38:22.485552+0000 mon.c (mon.2) 213 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:38:23.506 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:23 smithi088 bash[24895]: cephadm 2024-09-13T00:38:22.487319+0000 mgr.y (mgr.25064) 32 : cephadm [INF] Deploying daemon mgr.x on smithi088 2024-09-13T00:38:23.506 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:23 smithi088 bash[24895]: audit 2024-09-13T00:38:22.509348+0000 mon.a (mon.0) 1197 : audit [DBG] from='client.? 172.21.15.77:0/1922973511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:23.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[22596]: cluster 2024-09-13T00:38:22.217013+0000 mon.a (mon.0) 1195 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2024-09-13T00:38:23.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[22596]: cluster 2024-09-13T00:38:22.268812+0000 mgr.y (mgr.25064) 30 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 873 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s rd, 1004 KiB/s wr, 2 op/s 2024-09-13T00:38:23.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[22596]: cephadm 2024-09-13T00:38:22.480621+0000 mgr.y (mgr.25064) 31 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-13T00:38:23.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[22596]: audit 2024-09-13T00:38:22.481221+0000 mon.c (mon.2) 211 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:38:23.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[22596]: audit 2024-09-13T00:38:22.481986+0000 mon.a (mon.0) 1196 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:38:23.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[22596]: audit 2024-09-13T00:38:22.483767+0000 mon.c (mon.2) 212 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:38:23.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[22596]: audit 2024-09-13T00:38:22.485552+0000 mon.c (mon.2) 213 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:38:23.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[22596]: cephadm 2024-09-13T00:38:22.487319+0000 mgr.y (mgr.25064) 32 : cephadm [INF] Deploying daemon mgr.x on smithi088 2024-09-13T00:38:23.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[22596]: audit 2024-09-13T00:38:22.509348+0000 mon.a (mon.0) 1197 : audit [DBG] from='client.? 172.21.15.77:0/1922973511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:23.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[32040]: cluster 2024-09-13T00:38:22.217013+0000 mon.a (mon.0) 1195 : cluster [DBG] osdmap e257: 8 total, 8 up, 8 in 2024-09-13T00:38:23.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[32040]: cluster 2024-09-13T00:38:22.268812+0000 mgr.y (mgr.25064) 30 : cluster [DBG] pgmap v39: 137 pgs: 137 active+clean; 873 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.0 MiB/s rd, 1004 KiB/s wr, 2 op/s 2024-09-13T00:38:23.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[32040]: cephadm 2024-09-13T00:38:22.480621+0000 mgr.y (mgr.25064) 31 : cephadm [INF] Upgrade: Updating mgr.x 2024-09-13T00:38:23.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[32040]: audit 2024-09-13T00:38:22.481221+0000 mon.c (mon.2) 211 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:38:23.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[32040]: audit 2024-09-13T00:38:22.481986+0000 mon.a (mon.0) 1196 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:38:23.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[32040]: audit 2024-09-13T00:38:22.483767+0000 mon.c (mon.2) 212 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:38:23.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[32040]: audit 2024-09-13T00:38:22.485552+0000 mon.c (mon.2) 213 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:38:23.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[32040]: cephadm 2024-09-13T00:38:22.487319+0000 mgr.y (mgr.25064) 32 : cephadm [INF] Deploying daemon mgr.x on smithi088 2024-09-13T00:38:23.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:23 smithi077 bash[32040]: audit 2024-09-13T00:38:22.509348+0000 mon.a (mon.0) 1197 : audit [DBG] from='client.? 172.21.15.77:0/1922973511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:23.850 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:23 smithi088 bash[61791]: ts=2024-09-13T00:38:23.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:24.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:24 smithi077 bash[22596]: cluster 2024-09-13T00:38:23.233596+0000 mon.a (mon.0) 1198 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2024-09-13T00:38:24.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:24 smithi077 bash[32040]: cluster 2024-09-13T00:38:23.233596+0000 mon.a (mon.0) 1198 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2024-09-13T00:38:24.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:24 smithi088 bash[24895]: cluster 2024-09-13T00:38:23.233596+0000 mon.a (mon.0) 1198 : cluster [DBG] osdmap e258: 8 total, 8 up, 8 in 2024-09-13T00:38:24.835 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.AbortWithSnapshots (15505 ms) 2024-09-13T00:38:24.835 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.CloneV1Parent 2024-09-13T00:38:24.900 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:24 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:24.901 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:24 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:24.901 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:38:24 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:24.901 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:38:24 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:24.902 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:38:24 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:24.902 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 00:38:24 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:24.902 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:24 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:24.903 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:38:24 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:24.903 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:38:24 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:24.903 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:38:24.916 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 0~10 2024-09-13T00:38:24.927 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:38:25.229 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:24 smithi088 systemd[1]: Stopping Ceph mgr.x for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:38:25.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:25 smithi077 bash[32040]: cluster 2024-09-13T00:38:24.234886+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2024-09-13T00:38:25.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:25 smithi077 bash[32040]: cluster 2024-09-13T00:38:24.269634+0000 mgr.y (mgr.25064) 33 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 385 op/s 2024-09-13T00:38:25.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:25 smithi077 bash[32040]: audit 2024-09-13T00:38:24.893153+0000 mon.a (mon.0) 1200 : audit [DBG] from='client.? 172.21.15.77:0/3070067397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:25.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:25 smithi077 bash[22596]: cluster 2024-09-13T00:38:24.234886+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2024-09-13T00:38:25.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:25 smithi077 bash[22596]: cluster 2024-09-13T00:38:24.269634+0000 mgr.y (mgr.25064) 33 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 385 op/s 2024-09-13T00:38:25.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:25 smithi077 bash[22596]: audit 2024-09-13T00:38:24.893153+0000 mon.a (mon.0) 1200 : audit [DBG] from='client.? 172.21.15.77:0/3070067397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:25.549 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:25 smithi088 bash[24895]: cluster 2024-09-13T00:38:24.234886+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e259: 8 total, 8 up, 8 in 2024-09-13T00:38:25.549 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:25 smithi088 bash[24895]: cluster 2024-09-13T00:38:24.269634+0000 mgr.y (mgr.25064) 33 : cluster [DBG] pgmap v42: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 23 MiB/s wr, 385 op/s 2024-09-13T00:38:25.549 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:25 smithi088 bash[24895]: audit 2024-09-13T00:38:24.893153+0000 mon.a (mon.0) 1200 : audit [DBG] from='client.? 172.21.15.77:0/3070067397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:25.850 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:25 smithi088 bash[62071]: ceph-9218306c-7165-11ef-bceb-c7b262605968-mgr-x 2024-09-13T00:38:25.851 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:25 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-13T00:38:25.851 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:25 smithi088 bash[62132]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-mgr-x 2024-09-13T00:38:25.851 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:25 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-13T00:38:25.851 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:25 smithi088 systemd[1]: Stopped Ceph mgr.x for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:38:26.201 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap1 2024-09-13T00:38:26.245 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:26.246 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:26 smithi088 systemd[1]: Started Ceph mgr.x for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:38:26.246 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:38:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:26.246 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:26.246 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:38:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:26.247 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 00:38:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:26.247 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:26.247 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:38:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:26.248 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:38:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:26.248 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:38:25 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:26.333 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:38:26.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:26 smithi077 bash[22596]: cluster 2024-09-13T00:38:25.256786+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2024-09-13T00:38:26.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:26 smithi077 bash[22596]: audit 2024-09-13T00:38:26.046694+0000 mon.a (mon.0) 1202 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:26.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:26 smithi077 bash[22596]: audit 2024-09-13T00:38:26.059125+0000 mon.a (mon.0) 1203 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:26.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:26 smithi077 bash[22596]: audit 2024-09-13T00:38:26.062354+0000 mon.c (mon.2) 214 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:38:26.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:26 smithi077 bash[22596]: cluster 2024-09-13T00:38:26.182766+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2024-09-13T00:38:26.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:26 smithi077 bash[32040]: cluster 2024-09-13T00:38:25.256786+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2024-09-13T00:38:26.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:26 smithi077 bash[32040]: audit 2024-09-13T00:38:26.046694+0000 mon.a (mon.0) 1202 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:26.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:26 smithi077 bash[32040]: audit 2024-09-13T00:38:26.059125+0000 mon.a (mon.0) 1203 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:26.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:26 smithi077 bash[32040]: audit 2024-09-13T00:38:26.062354+0000 mon.c (mon.2) 214 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:38:26.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:26 smithi077 bash[32040]: cluster 2024-09-13T00:38:26.182766+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2024-09-13T00:38:26.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:26 smithi088 bash[24895]: cluster 2024-09-13T00:38:25.256786+0000 mon.a (mon.0) 1201 : cluster [DBG] osdmap e260: 8 total, 8 up, 8 in 2024-09-13T00:38:26.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:26 smithi088 bash[24895]: audit 2024-09-13T00:38:26.046694+0000 mon.a (mon.0) 1202 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:26.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:26 smithi088 bash[24895]: audit 2024-09-13T00:38:26.059125+0000 mon.a (mon.0) 1203 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:26.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:26 smithi088 bash[24895]: audit 2024-09-13T00:38:26.062354+0000 mon.c (mon.2) 214 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:38:26.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:26 smithi088 bash[24895]: cluster 2024-09-13T00:38:26.182766+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e261: 8 total, 8 up, 8 in 2024-09-13T00:38:27.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:27 smithi088 bash[24895]: cluster 2024-09-13T00:38:26.270445+0000 mgr.y (mgr.25064) 34 : cluster [DBG] pgmap v45: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 40 MiB/s rd, 22 MiB/s wr, 382 op/s 2024-09-13T00:38:27.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:27 smithi088 bash[24895]: audit 2024-09-13T00:38:26.847958+0000 mon.c (mon.2) 215 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:27.351 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:27 smithi088 bash[62189]: debug 2024-09-13T00:38:27.161+0000 7fc42414f640 1 -- 172.21.15.88:0/2970572734 <== mon.1 v2:172.21.15.88:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5607174f2d00 con 0x5607174e5400 2024-09-13T00:38:27.351 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:27 smithi088 bash[62189]: debug 2024-09-13T00:38:27.281+0000 7fc4269b8040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T00:38:27.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:27 smithi077 bash[22596]: cluster 2024-09-13T00:38:26.270445+0000 mgr.y (mgr.25064) 34 : cluster [DBG] pgmap v45: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 40 MiB/s rd, 22 MiB/s wr, 382 op/s 2024-09-13T00:38:27.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:27 smithi077 bash[22596]: audit 2024-09-13T00:38:26.847958+0000 mon.c (mon.2) 215 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:27.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:27 smithi077 bash[32040]: cluster 2024-09-13T00:38:26.270445+0000 mgr.y (mgr.25064) 34 : cluster [DBG] pgmap v45: 137 pgs: 137 active+clean; 945 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 40 MiB/s rd, 22 MiB/s wr, 382 op/s 2024-09-13T00:38:27.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:27 smithi077 bash[32040]: audit 2024-09-13T00:38:26.847958+0000 mon.c (mon.2) 215 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:27.713 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:38:27.922 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:38:28.523 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.CloneV1Parent (3688 ms) 2024-09-13T00:38:28.523 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.CloneV2Parent 2024-09-13T00:38:28.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:28 smithi077 bash[32040]: audit 2024-09-13T00:38:27.253999+0000 mon.c (mon.2) 216 : audit [DBG] from='client.? 172.21.15.77:0/3332449774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:28.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:28 smithi077 bash[32040]: cluster 2024-09-13T00:38:27.262284+0000 mon.a (mon.0) 1205 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2024-09-13T00:38:28.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:28 smithi077 bash[22596]: audit 2024-09-13T00:38:27.253999+0000 mon.c (mon.2) 216 : audit [DBG] from='client.? 172.21.15.77:0/3332449774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:28.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:28 smithi077 bash[22596]: cluster 2024-09-13T00:38:27.262284+0000 mon.a (mon.0) 1205 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2024-09-13T00:38:28.566 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:28 smithi088 bash[24895]: audit 2024-09-13T00:38:27.253999+0000 mon.c (mon.2) 216 : audit [DBG] from='client.? 172.21.15.77:0/3332449774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:28.566 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:28 smithi088 bash[24895]: cluster 2024-09-13T00:38:27.262284+0000 mon.a (mon.0) 1205 : cluster [DBG] osdmap e262: 8 total, 8 up, 8 in 2024-09-13T00:38:28.598 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:38:28.624 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 0~10 2024-09-13T00:38:28.647 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:38:28.850 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:28 smithi088 bash[62189]: debug 2024-09-13T00:38:28.561+0000 7fc4269b8040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T00:38:28.851 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:28 smithi088 bash[62189]: debug 2024-09-13T00:38:28.633+0000 7fc4269b8040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T00:38:28.851 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:28 smithi088 bash[62189]: debug 2024-09-13T00:38:28.785+0000 7fc4269b8040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T00:38:28.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:28 smithi088 bash[61791]: ts=2024-09-13T00:38:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:29.026 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:38:28] "GET /metrics HTTP/1.1" 200 38820 "" "Prometheus/2.43.0" 2024-09-13T00:38:29.569 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:29 smithi088 bash[24895]: cluster 2024-09-13T00:38:28.272174+0000 mgr.y (mgr.25064) 35 : cluster [DBG] pgmap v47: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 16 KiB/s wr, 225 op/s 2024-09-13T00:38:29.569 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:29 smithi088 bash[24895]: cluster 2024-09-13T00:38:28.279080+0000 mon.a (mon.0) 1206 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2024-09-13T00:38:29.569 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:29 smithi088 bash[62189]: debug 2024-09-13T00:38:29.429+0000 7fc4269b8040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T00:38:29.570 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:29 smithi088 bash[62189]: debug 2024-09-13T00:38:29.497+0000 7fc4269b8040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T00:38:29.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:29 smithi077 bash[22596]: cluster 2024-09-13T00:38:28.272174+0000 mgr.y (mgr.25064) 35 : cluster [DBG] pgmap v47: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 16 KiB/s wr, 225 op/s 2024-09-13T00:38:29.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:29 smithi077 bash[22596]: cluster 2024-09-13T00:38:28.279080+0000 mon.a (mon.0) 1206 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2024-09-13T00:38:29.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:29 smithi077 bash[32040]: cluster 2024-09-13T00:38:28.272174+0000 mgr.y (mgr.25064) 35 : cluster [DBG] pgmap v47: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 36 MiB/s rd, 16 KiB/s wr, 225 op/s 2024-09-13T00:38:29.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:29 smithi077 bash[32040]: cluster 2024-09-13T00:38:28.279080+0000 mon.a (mon.0) 1206 : cluster [DBG] osdmap e263: 8 total, 8 up, 8 in 2024-09-13T00:38:29.827 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:29 smithi088 bash[62189]: debug 2024-09-13T00:38:29.565+0000 7fc4269b8040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T00:38:29.827 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:29 smithi088 bash[62189]: debug 2024-09-13T00:38:29.749+0000 7fc4269b8040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T00:38:30.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:29 smithi088 bash[62189]: debug 2024-09-13T00:38:29.825+0000 7fc4269b8040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T00:38:30.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:29 smithi088 bash[62189]: debug 2024-09-13T00:38:29.889+0000 7fc4269b8040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T00:38:30.310 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap1 2024-09-13T00:38:30.419 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:30 smithi088 bash[62189]: debug 2024-09-13T00:38:30.141+0000 7fc4269b8040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T00:38:30.419 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:30 smithi088 bash[62189]: debug 2024-09-13T00:38:30.209+0000 7fc4269b8040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T00:38:30.419 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:30 smithi088 bash[24895]: cluster 2024-09-13T00:38:29.287007+0000 mon.a (mon.0) 1207 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2024-09-13T00:38:30.419 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:30 smithi088 bash[24895]: audit 2024-09-13T00:38:29.621113+0000 mon.a (mon.0) 1208 : audit [DBG] from='client.? 172.21.15.77:0/517604714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:30.446 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:38:30.757 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:30 smithi088 bash[62189]: debug 2024-09-13T00:38:30.417+0000 7fc4269b8040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T00:38:30.757 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:30 smithi088 bash[62189]: debug 2024-09-13T00:38:30.497+0000 7fc4269b8040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T00:38:30.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:30 smithi077 bash[22596]: cluster 2024-09-13T00:38:29.287007+0000 mon.a (mon.0) 1207 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2024-09-13T00:38:30.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:30 smithi077 bash[22596]: audit 2024-09-13T00:38:29.621113+0000 mon.a (mon.0) 1208 : audit [DBG] from='client.? 172.21.15.77:0/517604714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:30.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:30 smithi077 bash[32040]: cluster 2024-09-13T00:38:29.287007+0000 mon.a (mon.0) 1207 : cluster [DBG] osdmap e264: 8 total, 8 up, 8 in 2024-09-13T00:38:30.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:30 smithi077 bash[32040]: audit 2024-09-13T00:38:29.621113+0000 mon.a (mon.0) 1208 : audit [DBG] from='client.? 172.21.15.77:0/517604714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:31.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:30 smithi088 bash[62189]: debug 2024-09-13T00:38:30.753+0000 7fc4269b8040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T00:38:31.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:30 smithi088 bash[62189]: debug 2024-09-13T00:38:30.925+0000 7fc4269b8040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:38:31.401 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:31 smithi088 bash[62189]: debug 2024-09-13T00:38:31.101+0000 7fc4269b8040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T00:38:31.401 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:31 smithi088 bash[62189]: debug 2024-09-13T00:38:31.193+0000 7fc4269b8040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T00:38:31.402 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:31 smithi088 bash[24895]: cluster 2024-09-13T00:38:30.273062+0000 mgr.y (mgr.25064) 36 : cluster [DBG] pgmap v50: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 15 KiB/s wr, 222 op/s 2024-09-13T00:38:31.402 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:31 smithi088 bash[24895]: cluster 2024-09-13T00:38:30.297260+0000 mon.a (mon.0) 1209 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2024-09-13T00:38:31.402 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:31 smithi088 bash[24895]: cluster 2024-09-13T00:38:31.184660+0000 mon.a (mon.0) 1210 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2024-09-13T00:38:31.570 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:38:31.705 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:38:31.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:31 smithi077 bash[22596]: cluster 2024-09-13T00:38:30.273062+0000 mgr.y (mgr.25064) 36 : cluster [DBG] pgmap v50: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 15 KiB/s wr, 222 op/s 2024-09-13T00:38:31.776 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:31 smithi077 bash[22596]: cluster 2024-09-13T00:38:30.297260+0000 mon.a (mon.0) 1209 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2024-09-13T00:38:31.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:31 smithi077 bash[22596]: cluster 2024-09-13T00:38:31.184660+0000 mon.a (mon.0) 1210 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2024-09-13T00:38:31.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:31 smithi077 bash[32040]: cluster 2024-09-13T00:38:30.273062+0000 mgr.y (mgr.25064) 36 : cluster [DBG] pgmap v50: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 35 MiB/s rd, 15 KiB/s wr, 222 op/s 2024-09-13T00:38:31.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:31 smithi077 bash[32040]: cluster 2024-09-13T00:38:30.297260+0000 mon.a (mon.0) 1209 : cluster [DBG] osdmap e265: 8 total, 8 up, 8 in 2024-09-13T00:38:31.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:31 smithi077 bash[32040]: cluster 2024-09-13T00:38:31.184660+0000 mon.a (mon.0) 1210 : cluster [DBG] osdmap e266: 8 total, 8 up, 8 in 2024-09-13T00:38:31.850 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:31 smithi088 bash[62189]: debug 2024-09-13T00:38:31.397+0000 7fc4269b8040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:38:32.510 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.CloneV2Parent (3987 ms) 2024-09-13T00:38:32.510 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.CloneV1ParentAbort 2024-09-13T00:38:32.589 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:38:32.600 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:32 smithi088 bash[62189]: debug 2024-09-13T00:38:32.289+0000 7fc4269b8040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T00:38:32.600 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:32 smithi088 bash[62189]: debug 2024-09-13T00:38:32.373+0000 7fc4269b8040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T00:38:32.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:32 smithi088 bash[61791]: ts=2024-09-13T00:38:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:32.615 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 0~10 2024-09-13T00:38:32.637 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:38:32.938 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:32 smithi088 bash[62189]: debug 2024-09-13T00:38:32.617+0000 7fc4269b8040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T00:38:32.938 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:32 smithi088 bash[62189]: debug 2024-09-13T00:38:32.697+0000 7fc4269b8040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T00:38:33.194 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:33 smithi088 bash[62189]: debug 2024-09-13T00:38:33.089+0000 7fc4269b8040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T00:38:33.195 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:33 smithi088 bash[62189]: debug 2024-09-13T00:38:33.165+0000 7fc4269b8040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T00:38:33.195 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:32 smithi088 bash[24895]: audit 2024-09-13T00:38:31.943492+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:33.195 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:32 smithi088 bash[24895]: audit 2024-09-13T00:38:31.959353+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:33.195 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:32 smithi088 bash[24895]: audit 2024-09-13T00:38:32.024081+0000 mon.c (mon.2) 217 : audit [DBG] from='client.? 172.21.15.77:0/2277807511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:33.195 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:32 smithi088 bash[24895]: cluster 2024-09-13T00:38:32.274108+0000 mgr.y (mgr.25064) 37 : cluster [DBG] pgmap v53: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 2.5 KiB/s rd, 767 B/s wr, 5 op/s 2024-09-13T00:38:33.195 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:32 smithi088 bash[24895]: cluster 2024-09-13T00:38:32.306570+0000 mon.a (mon.0) 1213 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2024-09-13T00:38:33.195 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:32 smithi088 bash[24895]: audit 2024-09-13T00:38:32.673777+0000 mon.a (mon.0) 1214 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:33.195 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:32 smithi088 bash[24895]: audit 2024-09-13T00:38:32.686309+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:33.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:32 smithi077 bash[22596]: audit 2024-09-13T00:38:31.943492+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:33.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:32 smithi077 bash[22596]: audit 2024-09-13T00:38:31.959353+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:33.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:32 smithi077 bash[22596]: audit 2024-09-13T00:38:32.024081+0000 mon.c (mon.2) 217 : audit [DBG] from='client.? 172.21.15.77:0/2277807511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:33.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:32 smithi077 bash[22596]: cluster 2024-09-13T00:38:32.274108+0000 mgr.y (mgr.25064) 37 : cluster [DBG] pgmap v53: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 2.5 KiB/s rd, 767 B/s wr, 5 op/s 2024-09-13T00:38:33.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:32 smithi077 bash[22596]: cluster 2024-09-13T00:38:32.306570+0000 mon.a (mon.0) 1213 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2024-09-13T00:38:33.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:32 smithi077 bash[22596]: audit 2024-09-13T00:38:32.673777+0000 mon.a (mon.0) 1214 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:33.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:32 smithi077 bash[22596]: audit 2024-09-13T00:38:32.686309+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:33.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:32 smithi077 bash[32040]: audit 2024-09-13T00:38:31.943492+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:33.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:32 smithi077 bash[32040]: audit 2024-09-13T00:38:31.959353+0000 mon.a (mon.0) 1212 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:33.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:32 smithi077 bash[32040]: audit 2024-09-13T00:38:32.024081+0000 mon.c (mon.2) 217 : audit [DBG] from='client.? 172.21.15.77:0/2277807511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:33.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:32 smithi077 bash[32040]: cluster 2024-09-13T00:38:32.274108+0000 mgr.y (mgr.25064) 37 : cluster [DBG] pgmap v53: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 2.5 KiB/s rd, 767 B/s wr, 5 op/s 2024-09-13T00:38:33.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:32 smithi077 bash[32040]: cluster 2024-09-13T00:38:32.306570+0000 mon.a (mon.0) 1213 : cluster [DBG] osdmap e267: 8 total, 8 up, 8 in 2024-09-13T00:38:33.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:32 smithi077 bash[32040]: audit 2024-09-13T00:38:32.673777+0000 mon.a (mon.0) 1214 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:33.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:32 smithi077 bash[32040]: audit 2024-09-13T00:38:32.686309+0000 mon.a (mon.0) 1215 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:33.505 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:33 smithi088 bash[62189]: [13/Sep/2024:00:38:33] ENGINE Bus STARTING 2024-09-13T00:38:33.505 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:33 smithi088 bash[62189]: CherryPy Checker: 2024-09-13T00:38:33.505 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:33 smithi088 bash[62189]: The Application mounted at '' has an empty config. 2024-09-13T00:38:33.505 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:33 smithi088 bash[62189]: [13/Sep/2024:00:38:33] ENGINE Serving on http://:::9283 2024-09-13T00:38:33.505 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:33 smithi088 bash[62189]: [13/Sep/2024:00:38:33] ENGINE Bus STARTED 2024-09-13T00:38:33.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:33 smithi088 bash[61791]: ts=2024-09-13T00:38:33.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:34.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:33 smithi077 bash[22596]: cluster 2024-09-13T00:38:33.194890+0000 mon.a (mon.0) 1216 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T00:38:34.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:33 smithi077 bash[22596]: cluster 2024-09-13T00:38:33.195097+0000 mon.a (mon.0) 1217 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:38:34.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:33 smithi077 bash[22596]: audit 2024-09-13T00:38:33.197964+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.? 172.21.15.88:0/1001140111' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:38:34.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:33 smithi077 bash[22596]: audit 2024-09-13T00:38:33.198970+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.? 172.21.15.88:0/1001140111' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:38:34.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:33 smithi077 bash[22596]: audit 2024-09-13T00:38:33.200425+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.? 172.21.15.88:0/1001140111' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:38:34.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:33 smithi077 bash[22596]: audit 2024-09-13T00:38:33.201353+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.? 172.21.15.88:0/1001140111' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:38:34.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:33 smithi077 bash[22596]: cluster 2024-09-13T00:38:33.303906+0000 mon.a (mon.0) 1218 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2024-09-13T00:38:34.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:33 smithi077 bash[32040]: cluster 2024-09-13T00:38:33.194890+0000 mon.a (mon.0) 1216 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T00:38:34.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:33 smithi077 bash[32040]: cluster 2024-09-13T00:38:33.195097+0000 mon.a (mon.0) 1217 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:38:34.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:33 smithi077 bash[32040]: audit 2024-09-13T00:38:33.197964+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.? 172.21.15.88:0/1001140111' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:38:34.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:33 smithi077 bash[32040]: audit 2024-09-13T00:38:33.198970+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.? 172.21.15.88:0/1001140111' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:38:34.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:33 smithi077 bash[32040]: audit 2024-09-13T00:38:33.200425+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.? 172.21.15.88:0/1001140111' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:38:34.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:33 smithi077 bash[32040]: audit 2024-09-13T00:38:33.201353+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.? 172.21.15.88:0/1001140111' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:38:34.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:33 smithi077 bash[32040]: cluster 2024-09-13T00:38:33.303906+0000 mon.a (mon.0) 1218 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2024-09-13T00:38:34.331 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap1 2024-09-13T00:38:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:33 smithi088 bash[24895]: cluster 2024-09-13T00:38:33.194890+0000 mon.a (mon.0) 1216 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T00:38:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:33 smithi088 bash[24895]: cluster 2024-09-13T00:38:33.195097+0000 mon.a (mon.0) 1217 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:38:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:33 smithi088 bash[24895]: audit 2024-09-13T00:38:33.197964+0000 mon.b (mon.1) 87 : audit [DBG] from='mgr.? 172.21.15.88:0/1001140111' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:38:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:33 smithi088 bash[24895]: audit 2024-09-13T00:38:33.198970+0000 mon.b (mon.1) 88 : audit [DBG] from='mgr.? 172.21.15.88:0/1001140111' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:38:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:33 smithi088 bash[24895]: audit 2024-09-13T00:38:33.200425+0000 mon.b (mon.1) 89 : audit [DBG] from='mgr.? 172.21.15.88:0/1001140111' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:38:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:33 smithi088 bash[24895]: audit 2024-09-13T00:38:33.201353+0000 mon.b (mon.1) 90 : audit [DBG] from='mgr.? 172.21.15.88:0/1001140111' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:38:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:33 smithi088 bash[24895]: cluster 2024-09-13T00:38:33.303906+0000 mon.a (mon.0) 1218 : cluster [DBG] osdmap e268: 8 total, 8 up, 8 in 2024-09-13T00:38:34.460 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:38:34.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:34 smithi088 bash[61791]: ts=2024-09-13T00:38:34.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:34.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:34 smithi088 bash[61791]: ts=2024-09-13T00:38:34.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:35.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:34 smithi077 bash[22596]: cluster 2024-09-13T00:38:33.980989+0000 mon.a (mon.0) 1219 : cluster [DBG] mgrmap e32: y(active, since 37s), standbys: x 2024-09-13T00:38:35.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:34 smithi077 bash[22596]: cluster 2024-09-13T00:38:34.274920+0000 mgr.y (mgr.25064) 38 : cluster [DBG] pgmap v56: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 526 KiB/s rd, 27 KiB/s wr, 653 op/s 2024-09-13T00:38:35.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:34 smithi077 bash[22596]: cluster 2024-09-13T00:38:34.307135+0000 mon.a (mon.0) 1220 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2024-09-13T00:38:35.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:34 smithi077 bash[22596]: audit 2024-09-13T00:38:34.390442+0000 mon.a (mon.0) 1221 : audit [DBG] from='client.? 172.21.15.77:0/3792161301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:35.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:34 smithi077 bash[32040]: cluster 2024-09-13T00:38:33.980989+0000 mon.a (mon.0) 1219 : cluster [DBG] mgrmap e32: y(active, since 37s), standbys: x 2024-09-13T00:38:35.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:34 smithi077 bash[32040]: cluster 2024-09-13T00:38:34.274920+0000 mgr.y (mgr.25064) 38 : cluster [DBG] pgmap v56: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 526 KiB/s rd, 27 KiB/s wr, 653 op/s 2024-09-13T00:38:35.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:34 smithi077 bash[32040]: cluster 2024-09-13T00:38:34.307135+0000 mon.a (mon.0) 1220 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2024-09-13T00:38:35.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:34 smithi077 bash[32040]: audit 2024-09-13T00:38:34.390442+0000 mon.a (mon.0) 1221 : audit [DBG] from='client.? 172.21.15.77:0/3792161301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:35.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:34 smithi088 bash[24895]: cluster 2024-09-13T00:38:33.980989+0000 mon.a (mon.0) 1219 : cluster [DBG] mgrmap e32: y(active, since 37s), standbys: x 2024-09-13T00:38:35.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:34 smithi088 bash[24895]: cluster 2024-09-13T00:38:34.274920+0000 mgr.y (mgr.25064) 38 : cluster [DBG] pgmap v56: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 526 KiB/s rd, 27 KiB/s wr, 653 op/s 2024-09-13T00:38:35.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:34 smithi088 bash[24895]: cluster 2024-09-13T00:38:34.307135+0000 mon.a (mon.0) 1220 : cluster [DBG] osdmap e269: 8 total, 8 up, 8 in 2024-09-13T00:38:35.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:34 smithi088 bash[24895]: audit 2024-09-13T00:38:34.390442+0000 mon.a (mon.0) 1221 : audit [DBG] from='client.? 172.21.15.77:0/3792161301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:35.687 INFO:tasks.workunit.client.0.smithi077.stdout:migration_abort 2024-09-13T00:38:36.456 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.CloneV1ParentAbort (3946 ms) 2024-09-13T00:38:36.457 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.CloneV2ParentAbort 2024-09-13T00:38:36.516 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:38:36.531 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 0~10 2024-09-13T00:38:36.545 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:38:36.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:36 smithi088 bash[24895]: cluster 2024-09-13T00:38:35.312236+0000 mon.a (mon.0) 1222 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2024-09-13T00:38:36.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:36 smithi088 bash[24895]: cluster 2024-09-13T00:38:36.188267+0000 mon.a (mon.0) 1223 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2024-09-13T00:38:36.776 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:36 smithi077 bash[32040]: cluster 2024-09-13T00:38:35.312236+0000 mon.a (mon.0) 1222 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2024-09-13T00:38:36.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:36 smithi077 bash[32040]: cluster 2024-09-13T00:38:36.188267+0000 mon.a (mon.0) 1223 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2024-09-13T00:38:36.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:36 smithi077 bash[22596]: cluster 2024-09-13T00:38:35.312236+0000 mon.a (mon.0) 1222 : cluster [DBG] osdmap e270: 8 total, 8 up, 8 in 2024-09-13T00:38:36.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:36 smithi077 bash[22596]: cluster 2024-09-13T00:38:36.188267+0000 mon.a (mon.0) 1223 : cluster [DBG] osdmap e271: 8 total, 8 up, 8 in 2024-09-13T00:38:37.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:37 smithi088 bash[24895]: cluster 2024-09-13T00:38:36.275669+0000 mgr.y (mgr.25064) 39 : cluster [DBG] pgmap v60: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:38:37.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:37 smithi088 bash[24895]: audit 2024-09-13T00:38:36.759724+0000 mon.a (mon.0) 1224 : audit [DBG] from='client.? 172.21.15.77:0/3595682275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:37.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:37 smithi077 bash[22596]: cluster 2024-09-13T00:38:36.275669+0000 mgr.y (mgr.25064) 39 : cluster [DBG] pgmap v60: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:38:37.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:37 smithi077 bash[22596]: audit 2024-09-13T00:38:36.759724+0000 mon.a (mon.0) 1224 : audit [DBG] from='client.? 172.21.15.77:0/3595682275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:37.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:37 smithi077 bash[32040]: cluster 2024-09-13T00:38:36.275669+0000 mgr.y (mgr.25064) 39 : cluster [DBG] pgmap v60: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:38:37.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:37 smithi077 bash[32040]: audit 2024-09-13T00:38:36.759724+0000 mon.a (mon.0) 1224 : audit [DBG] from='client.? 172.21.15.77:0/3595682275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:38.362 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap1 2024-09-13T00:38:38.513 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:38:38.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:38 smithi088 bash[24895]: cluster 2024-09-13T00:38:37.328320+0000 mon.a (mon.0) 1225 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2024-09-13T00:38:38.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:38 smithi077 bash[22596]: cluster 2024-09-13T00:38:37.328320+0000 mon.a (mon.0) 1225 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2024-09-13T00:38:38.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:38:38] "GET /metrics HTTP/1.1" 200 38820 "" "Prometheus/2.43.0" 2024-09-13T00:38:38.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:38 smithi077 bash[32040]: cluster 2024-09-13T00:38:37.328320+0000 mon.a (mon.0) 1225 : cluster [DBG] osdmap e272: 8 total, 8 up, 8 in 2024-09-13T00:38:39.042 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:38 smithi088 bash[61791]: ts=2024-09-13T00:38:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:39.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:39 smithi088 bash[24895]: cluster 2024-09-13T00:38:38.276454+0000 mgr.y (mgr.25064) 40 : cluster [DBG] pgmap v62: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 308 KiB/s rd, 22 KiB/s wr, 403 op/s 2024-09-13T00:38:39.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:39 smithi088 bash[24895]: cluster 2024-09-13T00:38:38.345334+0000 mon.a (mon.0) 1226 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2024-09-13T00:38:39.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:39 smithi088 bash[24895]: audit 2024-09-13T00:38:39.113623+0000 mon.a (mon.0) 1227 : audit [DBG] from='client.? 172.21.15.77:0/1343740870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:39.702 INFO:tasks.workunit.client.0.smithi077.stdout:migration_abort 2024-09-13T00:38:39.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:39 smithi077 bash[22596]: cluster 2024-09-13T00:38:38.276454+0000 mgr.y (mgr.25064) 40 : cluster [DBG] pgmap v62: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 308 KiB/s rd, 22 KiB/s wr, 403 op/s 2024-09-13T00:38:39.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:39 smithi077 bash[22596]: cluster 2024-09-13T00:38:38.345334+0000 mon.a (mon.0) 1226 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2024-09-13T00:38:39.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:39 smithi077 bash[22596]: audit 2024-09-13T00:38:39.113623+0000 mon.a (mon.0) 1227 : audit [DBG] from='client.? 172.21.15.77:0/1343740870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:39.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:39 smithi077 bash[32040]: cluster 2024-09-13T00:38:38.276454+0000 mgr.y (mgr.25064) 40 : cluster [DBG] pgmap v62: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 308 KiB/s rd, 22 KiB/s wr, 403 op/s 2024-09-13T00:38:39.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:39 smithi077 bash[32040]: cluster 2024-09-13T00:38:38.345334+0000 mon.a (mon.0) 1226 : cluster [DBG] osdmap e273: 8 total, 8 up, 8 in 2024-09-13T00:38:39.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:39 smithi077 bash[32040]: audit 2024-09-13T00:38:39.113623+0000 mon.a (mon.0) 1227 : audit [DBG] from='client.? 172.21.15.77:0/1343740870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:40.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:40 smithi088 bash[24895]: cluster 2024-09-13T00:38:39.348455+0000 mon.a (mon.0) 1228 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2024-09-13T00:38:40.618 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.CloneV2ParentAbort (4161 ms) 2024-09-13T00:38:40.618 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach 2024-09-13T00:38:40.689 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:38:40.709 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 0~10 2024-09-13T00:38:40.725 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:38:40.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:40 smithi077 bash[22596]: cluster 2024-09-13T00:38:39.348455+0000 mon.a (mon.0) 1228 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2024-09-13T00:38:40.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:40 smithi077 bash[32040]: cluster 2024-09-13T00:38:39.348455+0000 mon.a (mon.0) 1228 : cluster [DBG] osdmap e274: 8 total, 8 up, 8 in 2024-09-13T00:38:41.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:41 smithi088 bash[24895]: cluster 2024-09-13T00:38:40.277364+0000 mgr.y (mgr.25064) 41 : cluster [DBG] pgmap v65: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 301 KiB/s rd, 22 KiB/s wr, 394 op/s 2024-09-13T00:38:41.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:41 smithi088 bash[24895]: cluster 2024-09-13T00:38:40.358007+0000 mon.a (mon.0) 1229 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2024-09-13T00:38:41.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:41 smithi088 bash[24895]: cluster 2024-09-13T00:38:41.189473+0000 mon.a (mon.0) 1230 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2024-09-13T00:38:41.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:41 smithi077 bash[22596]: cluster 2024-09-13T00:38:40.277364+0000 mgr.y (mgr.25064) 41 : cluster [DBG] pgmap v65: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 301 KiB/s rd, 22 KiB/s wr, 394 op/s 2024-09-13T00:38:41.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:41 smithi077 bash[22596]: cluster 2024-09-13T00:38:40.358007+0000 mon.a (mon.0) 1229 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2024-09-13T00:38:41.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:41 smithi077 bash[22596]: cluster 2024-09-13T00:38:41.189473+0000 mon.a (mon.0) 1230 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2024-09-13T00:38:41.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:41 smithi077 bash[32040]: cluster 2024-09-13T00:38:40.277364+0000 mgr.y (mgr.25064) 41 : cluster [DBG] pgmap v65: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 301 KiB/s rd, 22 KiB/s wr, 394 op/s 2024-09-13T00:38:41.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:41 smithi077 bash[32040]: cluster 2024-09-13T00:38:40.358007+0000 mon.a (mon.0) 1229 : cluster [DBG] osdmap e275: 8 total, 8 up, 8 in 2024-09-13T00:38:41.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:41 smithi077 bash[32040]: cluster 2024-09-13T00:38:41.189473+0000 mon.a (mon.0) 1230 : cluster [DBG] osdmap e276: 8 total, 8 up, 8 in 2024-09-13T00:38:42.205 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap1 2024-09-13T00:38:42.317 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:38:42.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[61791]: ts=2024-09-13T00:38:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:42.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:41.484802+0000 mon.c (mon.2) 218 : audit [DBG] from='client.? 172.21.15.77:0/3342586539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:42.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:41.848255+0000 mon.c (mon.2) 219 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:42.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:42.136644+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:42.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:42.153955+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:42.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:42.158475+0000 mon.c (mon.2) 220 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:38:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:42.160417+0000 mon.c (mon.2) 221 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:38:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:42.171578+0000 mon.a (mon.0) 1233 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: cluster 2024-09-13T00:38:42.191704+0000 mon.a (mon.0) 1234 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2024-09-13T00:38:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:42.305277+0000 mon.c (mon.2) 222 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:38:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:42.310040+0000 mon.c (mon.2) 223 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:38:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:42.317444+0000 mon.a (mon.0) 1235 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:42.322610+0000 mon.c (mon.2) 224 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T00:38:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:42.322991+0000 mon.a (mon.0) 1236 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T00:38:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:42.328850+0000 mon.a (mon.0) 1237 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T00:38:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:42.333660+0000 mon.c (mon.2) 225 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T00:38:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:42.334136+0000 mon.a (mon.0) 1238 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T00:38:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:42.341295+0000 mon.a (mon.0) 1239 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T00:38:42.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:42.343699+0000 mon.c (mon.2) 226 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T00:38:42.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:42 smithi088 bash[24895]: audit 2024-09-13T00:38:42.345202+0000 mon.c (mon.2) 227 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T00:38:42.692 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:41.484802+0000 mon.c (mon.2) 218 : audit [DBG] from='client.? 172.21.15.77:0/3342586539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:42.692 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:41.848255+0000 mon.c (mon.2) 219 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:42.692 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:42.136644+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:42.692 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:42.153955+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:42.692 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:42.158475+0000 mon.c (mon.2) 220 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:38:42.692 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:42.160417+0000 mon.c (mon.2) 221 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:38:42.692 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:42.171578+0000 mon.a (mon.0) 1233 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:42.692 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: cluster 2024-09-13T00:38:42.191704+0000 mon.a (mon.0) 1234 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2024-09-13T00:38:42.693 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:42.305277+0000 mon.c (mon.2) 222 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:38:42.693 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:42.310040+0000 mon.c (mon.2) 223 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:38:42.693 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:42.317444+0000 mon.a (mon.0) 1235 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:42.693 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:42.322610+0000 mon.c (mon.2) 224 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T00:38:42.693 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:42.322991+0000 mon.a (mon.0) 1236 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T00:38:42.693 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:42.328850+0000 mon.a (mon.0) 1237 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T00:38:42.693 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:42.333660+0000 mon.c (mon.2) 225 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T00:38:42.693 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:42.334136+0000 mon.a (mon.0) 1238 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T00:38:42.693 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:42.341295+0000 mon.a (mon.0) 1239 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T00:38:42.693 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:42.343699+0000 mon.c (mon.2) 226 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T00:38:42.693 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[22596]: audit 2024-09-13T00:38:42.345202+0000 mon.c (mon.2) 227 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T00:38:42.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:41.484802+0000 mon.c (mon.2) 218 : audit [DBG] from='client.? 172.21.15.77:0/3342586539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:42.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:41.848255+0000 mon.c (mon.2) 219 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:42.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:42.136644+0000 mon.a (mon.0) 1231 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:42.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:42.153955+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:42.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:42.158475+0000 mon.c (mon.2) 220 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:38:42.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:42.160417+0000 mon.c (mon.2) 221 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:38:42.694 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:42.171578+0000 mon.a (mon.0) 1233 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:42.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: cluster 2024-09-13T00:38:42.191704+0000 mon.a (mon.0) 1234 : cluster [DBG] osdmap e277: 8 total, 8 up, 8 in 2024-09-13T00:38:42.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:42.305277+0000 mon.c (mon.2) 222 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:38:42.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:42.310040+0000 mon.c (mon.2) 223 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:38:42.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:42.317444+0000 mon.a (mon.0) 1235 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:42.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:42.322610+0000 mon.c (mon.2) 224 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T00:38:42.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:42.322991+0000 mon.a (mon.0) 1236 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-13T00:38:42.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:42.328850+0000 mon.a (mon.0) 1237 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-13T00:38:42.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:42.333660+0000 mon.c (mon.2) 225 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T00:38:42.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:42.334136+0000 mon.a (mon.0) 1238 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-13T00:38:42.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:42.341295+0000 mon.a (mon.0) 1239 : audit [INF] from='mgr.25064 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-13T00:38:42.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:42.343699+0000 mon.c (mon.2) 226 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T00:38:42.695 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:42 smithi077 bash[32040]: audit 2024-09-13T00:38:42.345202+0000 mon.c (mon.2) 227 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-13T00:38:43.726 INFO:tasks.workunit.client.0.smithi077.stdout:migration_abort 2024-09-13T00:38:43.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:43 smithi077 bash[22596]: cluster 2024-09-13T00:38:42.278296+0000 mgr.y (mgr.25064) 42 : cluster [DBG] pgmap v69: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.8 KiB/s rd, 779 B/s wr, 8 op/s 2024-09-13T00:38:43.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:43 smithi077 bash[22596]: cephadm 2024-09-13T00:38:42.311348+0000 mgr.y (mgr.25064) 43 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-13T00:38:43.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:43 smithi077 bash[22596]: cephadm 2024-09-13T00:38:42.346228+0000 mgr.y (mgr.25064) 44 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-09-13T00:38:43.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:43 smithi077 bash[32040]: cluster 2024-09-13T00:38:42.278296+0000 mgr.y (mgr.25064) 42 : cluster [DBG] pgmap v69: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.8 KiB/s rd, 779 B/s wr, 8 op/s 2024-09-13T00:38:43.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:43 smithi077 bash[32040]: cephadm 2024-09-13T00:38:42.311348+0000 mgr.y (mgr.25064) 43 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-13T00:38:43.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:43 smithi077 bash[32040]: cephadm 2024-09-13T00:38:42.346228+0000 mgr.y (mgr.25064) 44 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-09-13T00:38:43.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:43 smithi088 bash[24895]: cluster 2024-09-13T00:38:42.278296+0000 mgr.y (mgr.25064) 42 : cluster [DBG] pgmap v69: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 2.8 KiB/s rd, 779 B/s wr, 8 op/s 2024-09-13T00:38:43.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:43 smithi088 bash[24895]: cephadm 2024-09-13T00:38:42.311348+0000 mgr.y (mgr.25064) 43 : cephadm [INF] Upgrade: Setting container_image for all mgr 2024-09-13T00:38:43.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:43 smithi088 bash[24895]: cephadm 2024-09-13T00:38:42.346228+0000 mgr.y (mgr.25064) 44 : cephadm [INF] Upgrade: It appears safe to stop mon.a 2024-09-13T00:38:43.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:43 smithi088 bash[61791]: ts=2024-09-13T00:38:43.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:44.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:44 smithi077 bash[32040]: cluster 2024-09-13T00:38:43.362795+0000 mon.a (mon.0) 1240 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2024-09-13T00:38:44.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:44 smithi077 bash[32040]: audit 2024-09-13T00:38:43.873711+0000 mon.c (mon.2) 228 : audit [DBG] from='client.? 172.21.15.77:0/3199714215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:44.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:44 smithi077 bash[22596]: cluster 2024-09-13T00:38:43.362795+0000 mon.a (mon.0) 1240 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2024-09-13T00:38:44.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:44 smithi077 bash[22596]: audit 2024-09-13T00:38:43.873711+0000 mon.c (mon.2) 228 : audit [DBG] from='client.? 172.21.15.77:0/3199714215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:44.551 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach (3933 ms) 2024-09-13T00:38:44.551 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixParentReattach 2024-09-13T00:38:44.605 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:38:44.631 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 0~10 2024-09-13T00:38:44.653 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:38:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:44 smithi088 bash[24895]: cluster 2024-09-13T00:38:43.362795+0000 mon.a (mon.0) 1240 : cluster [DBG] osdmap e278: 8 total, 8 up, 8 in 2024-09-13T00:38:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:44 smithi088 bash[24895]: audit 2024-09-13T00:38:43.873711+0000 mon.c (mon.2) 228 : audit [DBG] from='client.? 172.21.15.77:0/3199714215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:44.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:44 smithi088 bash[61791]: ts=2024-09-13T00:38:44.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:44.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:44 smithi088 bash[61791]: ts=2024-09-13T00:38:44.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:45.661 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:45 smithi077 bash[32040]: cluster 2024-09-13T00:38:44.278880+0000 mgr.y (mgr.25064) 45 : cluster [DBG] pgmap v71: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 279 KiB/s rd, 24 KiB/s wr, 364 op/s 2024-09-13T00:38:45.662 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:45 smithi077 bash[32040]: cluster 2024-09-13T00:38:44.382198+0000 mon.a (mon.0) 1241 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2024-09-13T00:38:45.662 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:45 smithi077 bash[32040]: audit 2024-09-13T00:38:45.248254+0000 mon.a (mon.0) 1242 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:45.662 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:45 smithi077 bash[32040]: audit 2024-09-13T00:38:45.249462+0000 mon.c (mon.2) 229 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:38:45.662 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:45 smithi077 bash[32040]: audit 2024-09-13T00:38:45.250363+0000 mon.c (mon.2) 230 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:38:45.662 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:45 smithi077 bash[32040]: audit 2024-09-13T00:38:45.251390+0000 mon.c (mon.2) 231 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:38:45.662 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:45 smithi077 bash[22596]: cluster 2024-09-13T00:38:44.278880+0000 mgr.y (mgr.25064) 45 : cluster [DBG] pgmap v71: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 279 KiB/s rd, 24 KiB/s wr, 364 op/s 2024-09-13T00:38:45.662 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:45 smithi077 bash[22596]: cluster 2024-09-13T00:38:44.382198+0000 mon.a (mon.0) 1241 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2024-09-13T00:38:45.662 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:45 smithi077 bash[22596]: audit 2024-09-13T00:38:45.248254+0000 mon.a (mon.0) 1242 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:45.662 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:45 smithi077 bash[22596]: audit 2024-09-13T00:38:45.249462+0000 mon.c (mon.2) 229 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:38:45.662 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:45 smithi077 bash[22596]: audit 2024-09-13T00:38:45.250363+0000 mon.c (mon.2) 230 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:38:45.662 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:45 smithi077 bash[22596]: audit 2024-09-13T00:38:45.251390+0000 mon.c (mon.2) 231 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:38:45.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:45 smithi088 bash[24895]: cluster 2024-09-13T00:38:44.278880+0000 mgr.y (mgr.25064) 45 : cluster [DBG] pgmap v71: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 279 KiB/s rd, 24 KiB/s wr, 364 op/s 2024-09-13T00:38:45.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:45 smithi088 bash[24895]: cluster 2024-09-13T00:38:44.382198+0000 mon.a (mon.0) 1241 : cluster [DBG] osdmap e279: 8 total, 8 up, 8 in 2024-09-13T00:38:45.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:45 smithi088 bash[24895]: audit 2024-09-13T00:38:45.248254+0000 mon.a (mon.0) 1242 : audit [INF] from='mgr.25064 ' entity='mgr.y' 2024-09-13T00:38:45.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:45 smithi088 bash[24895]: audit 2024-09-13T00:38:45.249462+0000 mon.c (mon.2) 229 : audit [INF] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:38:45.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:45 smithi088 bash[24895]: audit 2024-09-13T00:38:45.250363+0000 mon.c (mon.2) 230 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:38:45.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:45 smithi088 bash[24895]: audit 2024-09-13T00:38:45.251390+0000 mon.c (mon.2) 231 : audit [DBG] from='mgr.25064 172.21.15.77:0/3811485692' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:38:46.210 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap1 2024-09-13T00:38:46.307 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:38:46.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:46 smithi077 bash[32040]: cephadm 2024-09-13T00:38:45.240922+0000 mgr.y (mgr.25064) 46 : cephadm [INF] Upgrade: Updating mon.a 2024-09-13T00:38:46.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:46 smithi077 bash[32040]: cephadm 2024-09-13T00:38:45.253190+0000 mgr.y (mgr.25064) 47 : cephadm [INF] Deploying daemon mon.a on smithi077 2024-09-13T00:38:46.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:46 smithi077 bash[32040]: cluster 2024-09-13T00:38:45.393065+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2024-09-13T00:38:46.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:46 smithi077 bash[32040]: cluster 2024-09-13T00:38:46.192310+0000 mon.a (mon.0) 1244 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2024-09-13T00:38:46.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:46 smithi077 bash[32040]: audit 2024-09-13T00:38:46.248715+0000 mon.c (mon.2) 232 : audit [DBG] from='client.? 172.21.15.77:0/2556452077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:46.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:46 smithi077 bash[22596]: cephadm 2024-09-13T00:38:45.240922+0000 mgr.y (mgr.25064) 46 : cephadm [INF] Upgrade: Updating mon.a 2024-09-13T00:38:46.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:46 smithi077 bash[22596]: cephadm 2024-09-13T00:38:45.253190+0000 mgr.y (mgr.25064) 47 : cephadm [INF] Deploying daemon mon.a on smithi077 2024-09-13T00:38:46.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:46 smithi077 bash[22596]: cluster 2024-09-13T00:38:45.393065+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2024-09-13T00:38:46.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:46 smithi077 bash[22596]: cluster 2024-09-13T00:38:46.192310+0000 mon.a (mon.0) 1244 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2024-09-13T00:38:46.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:46 smithi077 bash[22596]: audit 2024-09-13T00:38:46.248715+0000 mon.c (mon.2) 232 : audit [DBG] from='client.? 172.21.15.77:0/2556452077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:46.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:46 smithi088 bash[24895]: cephadm 2024-09-13T00:38:45.240922+0000 mgr.y (mgr.25064) 46 : cephadm [INF] Upgrade: Updating mon.a 2024-09-13T00:38:46.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:46 smithi088 bash[24895]: cephadm 2024-09-13T00:38:45.253190+0000 mgr.y (mgr.25064) 47 : cephadm [INF] Deploying daemon mon.a on smithi077 2024-09-13T00:38:46.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:46 smithi088 bash[24895]: cluster 2024-09-13T00:38:45.393065+0000 mon.a (mon.0) 1243 : cluster [DBG] osdmap e280: 8 total, 8 up, 8 in 2024-09-13T00:38:46.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:46 smithi088 bash[24895]: cluster 2024-09-13T00:38:46.192310+0000 mon.a (mon.0) 1244 : cluster [DBG] osdmap e281: 8 total, 8 up, 8 in 2024-09-13T00:38:46.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:46 smithi088 bash[24895]: audit 2024-09-13T00:38:46.248715+0000 mon.c (mon.2) 232 : audit [DBG] from='client.? 172.21.15.77:0/2556452077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:47.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:47 smithi077 bash[22596]: cluster 2024-09-13T00:38:46.279317+0000 mgr.y (mgr.25064) 48 : cluster [DBG] pgmap v75: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 276 KiB/s rd, 23 KiB/s wr, 355 op/s 2024-09-13T00:38:47.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:47 smithi077 bash[32040]: cluster 2024-09-13T00:38:46.279317+0000 mgr.y (mgr.25064) 48 : cluster [DBG] pgmap v75: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 276 KiB/s rd, 23 KiB/s wr, 355 op/s 2024-09-13T00:38:47.808 INFO:tasks.workunit.client.0.smithi077.stdout:migration_abort 2024-09-13T00:38:47.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:47 smithi088 bash[24895]: cluster 2024-09-13T00:38:46.279317+0000 mgr.y (mgr.25064) 48 : cluster [DBG] pgmap v75: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 276 KiB/s rd, 23 KiB/s wr, 355 op/s 2024-09-13T00:38:48.264 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:38:48 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:48.265 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:38:48 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:48.265 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:38:48 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:48.265 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:38:48 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:48.266 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:38:48 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:48.266 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:48 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:48.266 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:48 smithi077 systemd[1]: Stopping Ceph mon.a for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:38:48.266 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:38:48 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:48.267 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:48 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:48.267 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:48 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:48.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:48 smithi077 bash[22596]: debug 2024-09-13T00:38:48.266+0000 7f651cc32640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T00:38:48.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:48 smithi077 bash[22596]: debug 2024-09-13T00:38:48.266+0000 7f651cc32640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-13T00:38:48.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:38:48] "GET /metrics HTTP/1.1" 200 38816 "" "Prometheus/2.43.0" 2024-09-13T00:38:49.101 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:48 smithi088 bash[61791]: ts=2024-09-13T00:38:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:49.199 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:48 smithi077 bash[80541]: ceph-9218306c-7165-11ef-bceb-c7b262605968-mon-a 2024-09-13T00:38:49.473 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:49.473 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:38:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:49.473 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:38:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:49.474 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:38:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:49.474 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:38:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:49.474 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:38:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:49.474 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:38:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:49.475 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:49.475 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:49 smithi077 bash[80625]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-mon-a 2024-09-13T00:38:49.475 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:49 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T00:38:49.475 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:49 smithi077 systemd[1]: Stopped Ceph mon.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:38:49.475 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:38:49.475 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:49 smithi077 systemd[1]: Started Ceph mon.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:38:51.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.786+0000 7fc7a0feab00 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-13T00:38:51.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.786+0000 7fc7a0feab00 0 ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable), process ceph-mon, pid 7 2024-09-13T00:38:51.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.786+0000 7fc7a0feab00 0 pidfile_write: ignore empty --pid-file 2024-09-13T00:38:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.786+0000 7fc7a0feab00 0 load: jerasure load: lrc 2024-09-13T00:38:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: RocksDB version: 7.9.2 2024-09-13T00:38:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Git sha 0 2024-09-13T00:38:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Compile date 2024-09-10 15:58:06 2024-09-13T00:38:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: DB SUMMARY 2024-09-13T00:38:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: DB Session ID: 7L8PH4860SI5EA1VF8CM 2024-09-13T00:38:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: CURRENT file: CURRENT 2024-09-13T00:38:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: IDENTITY file: IDENTITY 2024-09-13T00:38:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: MANIFEST file: MANIFEST-000015 size: 1741 Bytes 2024-09-13T00:38:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000042.sst 2024-09-13T00:38:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000040.log size: 3543226 ; 2024-09-13T00:38:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.error_if_exists: 0 2024-09-13T00:38:51.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.create_if_missing: 0 2024-09-13T00:38:51.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.paranoid_checks: 1 2024-09-13T00:38:51.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T00:38:51.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T00:38:51.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T00:38:51.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.env: 0x5584041fdbc0 2024-09-13T00:38:51.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.fs: PosixFileSystem 2024-09-13T00:38:51.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.info_log: 0x558405a3ebe0 2024-09-13T00:38:51.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-13T00:38:51.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.statistics: (nil) 2024-09-13T00:38:51.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.use_fsync: 0 2024-09-13T00:38:51.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_log_file_size: 0 2024-09-13T00:38:51.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T00:38:51.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T00:38:51.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-13T00:38:51.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-13T00:38:51.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.allow_fallocate: 1 2024-09-13T00:38:51.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-13T00:38:51.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-13T00:38:51.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.use_direct_reads: 0 2024-09-13T00:38:51.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T00:38:51.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.create_missing_column_families: 0 2024-09-13T00:38:51.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.db_log_dir: 2024-09-13T00:38:51.031 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.wal_dir: 2024-09-13T00:38:51.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T00:38:51.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T00:38:51.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T00:38:51.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T00:38:51.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T00:38:51.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T00:38:51.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.advise_random_on_open: 1 2024-09-13T00:38:51.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-13T00:38:51.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.write_buffer_manager: 0x558405a4c320 2024-09-13T00:38:51.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T00:38:51.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T00:38:51.032 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T00:38:51.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.rate_limiter: (nil) 2024-09-13T00:38:51.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T00:38:51.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-13T00:38:51.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-13T00:38:51.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-13T00:38:51.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.unordered_write: 0 2024-09-13T00:38:51.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T00:38:51.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T00:38:51.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T00:38:51.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T00:38:51.033 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.row_cache: None 2024-09-13T00:38:51.034 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.wal_filter: None 2024-09-13T00:38:51.034 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T00:38:51.034 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-13T00:38:51.034 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.two_write_queues: 0 2024-09-13T00:38:51.034 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.manual_wal_flush: 0 2024-09-13T00:38:51.034 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.wal_compression: 0 2024-09-13T00:38:51.034 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.atomic_flush: 0 2024-09-13T00:38:51.034 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T00:38:51.034 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T00:38:51.034 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T00:38:51.034 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.log_readahead_size: 0 2024-09-13T00:38:51.034 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T00:38:51.034 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-13T00:38:51.035 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T00:38:51.035 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T00:38:51.035 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-13T00:38:51.035 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-13T00:38:51.035 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T00:38:51.035 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_background_jobs: 2 2024-09-13T00:38:51.035 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_background_compactions: -1 2024-09-13T00:38:51.035 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_subcompactions: 1 2024-09-13T00:38:51.035 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T00:38:51.035 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T00:38:51.035 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T00:38:51.035 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_total_wal_size: 0 2024-09-13T00:38:51.035 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T00:38:51.036 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T00:38:51.036 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T00:38:51.036 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T00:38:51.036 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_open_files: -1 2024-09-13T00:38:51.036 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.bytes_per_sync: 0 2024-09-13T00:38:51.036 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T00:38:51.036 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T00:38:51.036 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-13T00:38:51.036 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_background_flushes: -1 2024-09-13T00:38:51.036 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Compression algorithms supported: 2024-09-13T00:38:51.036 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: kZSTD supported: 0 2024-09-13T00:38:51.036 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: kXpressCompression supported: 0 2024-09-13T00:38:51.036 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: kBZip2Compression supported: 0 2024-09-13T00:38:51.036 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T00:38:51.037 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: kLZ4Compression supported: 1 2024-09-13T00:38:51.037 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: kZlibCompression supported: 1 2024-09-13T00:38:51.037 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-13T00:38:51.037 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: kSnappyCompression supported: 1 2024-09-13T00:38:51.037 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T00:38:51.037 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T00:38:51.037 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-13T00:38:51.037 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T00:38:51.037 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T00:38:51.037 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.merge_operator: 2024-09-13T00:38:51.037 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compaction_filter: None 2024-09-13T00:38:51.037 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compaction_filter_factory: None 2024-09-13T00:38:51.037 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-13T00:38:51.037 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T00:38:51.038 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T00:38:51.038 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558405a3e7e0) 2024-09-13T00:38:51.038 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: cache_index_and_filter_blocks: 1 2024-09-13T00:38:51.038 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T00:38:51.038 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T00:38:51.038 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: pin_top_level_index_and_filter: 1 2024-09-13T00:38:51.038 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: index_type: 0 2024-09-13T00:38:51.038 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: data_block_index_type: 0 2024-09-13T00:38:51.038 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: index_shortening: 1 2024-09-13T00:38:51.038 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: data_block_hash_table_util_ratio: 0.750000 2024-09-13T00:38:51.038 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: checksum: 4 2024-09-13T00:38:51.038 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: no_block_cache: 0 2024-09-13T00:38:51.038 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: block_cache: 0x558405a351f0 2024-09-13T00:38:51.038 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: block_cache_name: BinnedLRUCache 2024-09-13T00:38:51.038 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: block_cache_options: 2024-09-13T00:38:51.039 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: capacity : 536870912 2024-09-13T00:38:51.039 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: num_shard_bits : 4 2024-09-13T00:38:51.039 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: strict_capacity_limit : 0 2024-09-13T00:38:51.039 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: high_pri_pool_ratio: 0.000 2024-09-13T00:38:51.039 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: block_cache_compressed: (nil) 2024-09-13T00:38:51.039 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: persistent_cache: (nil) 2024-09-13T00:38:51.039 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: block_size: 4096 2024-09-13T00:38:51.039 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: block_size_deviation: 10 2024-09-13T00:38:51.039 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: block_restart_interval: 16 2024-09-13T00:38:51.039 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: index_block_restart_interval: 1 2024-09-13T00:38:51.040 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: metadata_block_size: 4096 2024-09-13T00:38:51.040 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: partition_filters: 0 2024-09-13T00:38:51.040 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: use_delta_encoding: 1 2024-09-13T00:38:51.040 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: filter_policy: bloomfilter 2024-09-13T00:38:51.040 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: whole_key_filtering: 1 2024-09-13T00:38:51.040 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: verify_compression: 0 2024-09-13T00:38:51.040 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: read_amp_bytes_per_bit: 0 2024-09-13T00:38:51.040 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: format_version: 5 2024-09-13T00:38:51.040 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: enable_index_compression: 1 2024-09-13T00:38:51.040 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: block_align: 0 2024-09-13T00:38:51.040 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: max_auto_readahead_size: 262144 2024-09-13T00:38:51.040 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: prepopulate_block_cache: 0 2024-09-13T00:38:51.040 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: initial_auto_readahead_size: 8192 2024-09-13T00:38:51.040 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: num_file_reads_for_auto_readahead: 2 2024-09-13T00:38:51.041 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-13T00:38:51.041 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-13T00:38:51.041 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compression: NoCompression 2024-09-13T00:38:51.041 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-13T00:38:51.041 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-13T00:38:51.041 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T00:38:51.041 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.num_levels: 7 2024-09-13T00:38:51.041 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T00:38:51.041 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T00:38:51.041 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T00:38:51.041 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T00:38:51.041 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T00:38:51.041 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T00:38:51.042 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T00:38:51.042 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T00:38:51.042 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T00:38:51.042 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T00:38:51.042 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T00:38:51.042 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T00:38:51.042 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T00:38:51.042 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compression_opts.level: 32767 2024-09-13T00:38:51.042 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-13T00:38:51.042 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T00:38:51.042 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T00:38:51.042 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T00:38:51.042 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T00:38:51.042 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compression_opts.enabled: false 2024-09-13T00:38:51.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T00:38:51.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T00:38:51.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T00:38:51.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T00:38:51.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-13T00:38:51.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T00:38:51.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T00:38:51.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T00:38:51.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T00:38:51.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T00:38:51.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T00:38:51.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T00:38:51.043 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T00:38:51.044 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T00:38:51.044 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T00:38:51.044 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T00:38:51.044 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T00:38:51.044 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T00:38:51.044 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T00:38:51.044 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.arena_block_size: 1048576 2024-09-13T00:38:51.044 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T00:38:51.044 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T00:38:51.044 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-13T00:38:51.044 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T00:38:51.044 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T00:38:51.045 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T00:38:51.045 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T00:38:51.045 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T00:38:51.045 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T00:38:51.046 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T00:38:51.046 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T00:38:51.046 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T00:38:51.046 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T00:38:51.046 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T00:38:51.046 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.inplace_update_support: 0 2024-09-13T00:38:51.046 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T00:38:51.046 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T00:38:51.046 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T00:38:51.046 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T00:38:51.046 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.bloom_locality: 0 2024-09-13T00:38:51.047 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.max_successive_merges: 0 2024-09-13T00:38:51.047 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T00:38:51.047 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-13T00:38:51.047 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.force_consistency_checks: 1 2024-09-13T00:38:51.047 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-13T00:38:51.047 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.ttl: 2592000 2024-09-13T00:38:51.047 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T00:38:51.047 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T00:38:51.047 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T00:38:51.047 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.enable_blob_files: false 2024-09-13T00:38:51.047 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.min_blob_size: 0 2024-09-13T00:38:51.047 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.blob_file_size: 268435456 2024-09-13T00:38:51.048 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T00:38:51.048 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T00:38:51.048 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T00:38:51.048 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T00:38:51.048 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T00:38:51.048 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.blob_file_starting_level: 0 2024-09-13T00:38:51.048 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T00:38:51.048 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc79d5c2640 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 42.sst 2024-09-13T00:38:51.048 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T00:38:51.048 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 44, last_sequence is 17933, log_number is 40,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T00:38:51.048 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 40 2024-09-13T00:38:51.048 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 40b38235-3fb2-48fb-91d5-ab45c7566701 2024-09-13T00:38:51.048 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726187930792425, "job": 1, "event": "recovery_started", "wal_files": [40]} 2024-09-13T00:38:51.049 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.790+0000 7fc7a0feab00 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #40 mode 2 2024-09-13T00:38:51.049 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.802+0000 7fc7a0feab00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726187930803039, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 45, "file_size": 3371665, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 17934, "largest_seqno": 19582, "table_properties": {"data_size": 3362642, "index_size": 6121, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1861, "raw_key_size": 16912, "raw_average_key_size": 23, "raw_value_size": 3346156, "raw_average_value_size": 4647, "num_data_blocks": 267, "num_entries": 720, "num_filter_entries": 720, "num_deletions": 6, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726187930, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "40b38235-3fb2-48fb-91d5-ab45c7566701", "db_session_id": "7L8PH4860SI5EA1VF8CM", "orig_file_number": 45, "seqno_to_time_mapping": "N/A"}} 2024-09-13T00:38:51.049 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.802+0000 7fc7a0feab00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726187930803180, "job": 1, "event": "recovery_finished"} 2024-09-13T00:38:51.049 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.802+0000 7fc7a0feab00 4 rocksdb: [db/version_set.cc:5047] Creating manifest 47 2024-09-13T00:38:51.049 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.802+0000 7fc7a0feab00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T00:38:51.049 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.806+0000 7fc7a0feab00 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000040.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T00:38:51.049 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.806+0000 7fc7a0feab00 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x558405a5ee00 2024-09-13T00:38:51.049 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.806+0000 7fc7a0feab00 4 rocksdb: DB pointer 0x558405b56000 2024-09-13T00:38:51.049 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.806+0000 7fc796db5640 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T00:38:51.049 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.806+0000 7fc796db5640 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T00:38:51.049 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: ** DB Stats ** 2024-09-13T00:38:51.049 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:38:51.049 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T00:38:51.050 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T00:38:51.050 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T00:38:51.050 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T00:38:51.050 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T00:38:51.050 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T00:38:51.050 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: ** Compaction Stats [default] ** 2024-09-13T00:38:51.050 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T00:38:51.050 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T00:38:51.050 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: L0 1/0 3.22 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 630.2 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-13T00:38:51.051 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: L6 1/0 8.15 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T00:38:51.051 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Sum 2/0 11.36 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 630.2 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-13T00:38:51.051 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 630.2 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-13T00:38:51.051 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: ** Compaction Stats [default] ** 2024-09-13T00:38:51.051 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T00:38:51.051 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:38:51.051 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 630.2 0.01 0.00 1 0.005 0 0 0.0 0.0 2024-09-13T00:38:51.051 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T00:38:51.051 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:38:51.051 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Flush(GB): cumulative 0.003, interval 0.003 2024-09-13T00:38:51.051 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T00:38:51.051 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T00:38:51.051 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T00:38:51.052 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T00:38:51.052 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Cumulative compaction: 0.00 GB write, 204.99 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:38:51.052 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Interval compaction: 0.00 GB write, 204.99 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:38:51.052 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T00:38:51.052 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Block cache BinnedLRUCache@0x558405a351f0#7 capacity: 512.00 MB usage: 8.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.6e-05 secs_since: 0 2024-09-13T00:38:51.052 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Block cache entry stats(count,size,portion): FilterBlock(1,2.03 KB,0.00038743%) IndexBlock(1,6.08 KB,0.00115931%) Misc(1,0.00 KB,0%) 2024-09-13T00:38:51.052 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T00:38:51.052 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.806+0000 7fc7a0feab00 0 starting mon.a rank 0 at public addrs [v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0] at bind addrs [v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:38:51.052 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.806+0000 7fc7a0feab00 1 mon.a@-1(???) e3 preinit fsid 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:38:51.052 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.806+0000 7fc7a0feab00 0 mon.a@-1(???).mds e9 new map 2024-09-13T00:38:51.052 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.806+0000 7fc7a0feab00 0 mon.a@-1(???).mds e9 print_map 2024-09-13T00:38:51.052 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: e9 2024-09-13T00:38:51.053 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T00:38:51.053 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T00:38:51.053 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: legacy client fscid: 1 2024-09-13T00:38:51.053 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: 2024-09-13T00:38:51.053 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Filesystem 'foo' (1) 2024-09-13T00:38:51.053 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: fs_name foo 2024-09-13T00:38:51.053 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: epoch 9 2024-09-13T00:38:51.053 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T00:38:51.053 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: created 2024-09-13T00:32:23.134111+0000 2024-09-13T00:38:51.053 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: modified 2024-09-13T00:32:33.870108+0000 2024-09-13T00:38:51.053 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: tableserver 0 2024-09-13T00:38:51.053 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: root 0 2024-09-13T00:38:51.053 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: session_timeout 60 2024-09-13T00:38:51.053 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: session_autoclose 300 2024-09-13T00:38:51.054 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: max_file_size 1099511627776 2024-09-13T00:38:51.054 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: max_xattr_size 65536 2024-09-13T00:38:51.054 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: required_client_features {} 2024-09-13T00:38:51.054 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: last_failure 0 2024-09-13T00:38:51.054 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: last_failure_osd_epoch 63 2024-09-13T00:38:51.054 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T00:38:51.054 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: max_mds 1 2024-09-13T00:38:51.054 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: in 0 2024-09-13T00:38:51.054 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: up {0=24349} 2024-09-13T00:38:51.054 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: failed 2024-09-13T00:38:51.054 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: damaged 2024-09-13T00:38:51.054 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: stopped 2024-09-13T00:38:51.055 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: data_pools [4] 2024-09-13T00:38:51.055 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: metadata_pool 3 2024-09-13T00:38:51.055 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: inline_data disabled 2024-09-13T00:38:51.055 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: balancer 2024-09-13T00:38:51.055 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: bal_rank_mask -1 2024-09-13T00:38:51.055 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: standby_count_wanted 1 2024-09-13T00:38:51.055 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: [mds.foo.smithi088.krmowh{0:24349} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.88:6832/1144032959,v1:172.21.15.88:6833/1144032959] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T00:38:51.055 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: 2024-09-13T00:38:51.055 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: 2024-09-13T00:38:51.055 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: Standby daemons: 2024-09-13T00:38:51.055 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: 2024-09-13T00:38:51.055 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: [mds.foo.smithi077.bigbbf{-1:14604} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.77:6834/3765041927,v1:172.21.15.77:6835/3765041927] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T00:38:51.055 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.806+0000 7fc7a0feab00 0 mon.a@-1(???).osd e282 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T00:38:51.055 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.806+0000 7fc7a0feab00 0 mon.a@-1(???).osd e282 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T00:38:51.056 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.806+0000 7fc7a0feab00 0 mon.a@-1(???).osd e282 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T00:38:51.056 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.806+0000 7fc7a0feab00 0 mon.a@-1(???).osd e282 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T00:38:51.056 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:50 smithi077 bash[80681]: debug 2024-09-13T00:38:50.806+0000 7fc7a0feab00 1 mon.a@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-09-13T00:38:52.479 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixParentReattach (7927 ms) 2024-09-13T00:38:52.479 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.CloneV1ParentAbortRelinkNotNeeded 2024-09-13T00:38:52.530 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:38:52.555 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 0~10 2024-09-13T00:38:52.571 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:38:52.600 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[62189]: ignoring --setuser ceph since I am not root 2024-09-13T00:38:52.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[62189]: ignoring --setgroup ceph since I am not root 2024-09-13T00:38:52.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[62189]: debug 2024-09-13T00:38:52.481+0000 7f695fbf0040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T00:38:52.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[61791]: ts=2024-09-13T00:38:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:52.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[24895]: cluster 2024-09-13T00:38:48.279731+0000 mgr.y (mgr.25064) 49 : cluster [DBG] pgmap v77: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 324 KiB/s rd, 19 KiB/s wr, 413 op/s 2024-09-13T00:38:52.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[24895]: audit 2024-09-13T00:38:48.605678+0000 mon.c (mon.2) 233 : audit [DBG] from='client.? 172.21.15.77:0/3167492330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:52.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[24895]: cluster 2024-09-13T00:38:50.280456+0000 mgr.y (mgr.25064) 50 : cluster [DBG] pgmap v78: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 220 KiB/s rd, 13 KiB/s wr, 280 op/s 2024-09-13T00:38:52.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[24895]: audit 2024-09-13T00:38:50.970260+0000 mon.c (mon.2) 234 : audit [DBG] from='client.? 172.21.15.77:0/2141773463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[24895]: cluster 2024-09-13T00:38:51.424459+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-09-13T00:38:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[24895]: cluster 2024-09-13T00:38:52.248763+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T00:38:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[24895]: cluster 2024-09-13T00:38:52.257963+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:38:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[24895]: cluster 2024-09-13T00:38:52.275698+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:38:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[24895]: cluster 2024-09-13T00:38:52.275753+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2024-09-13T00:38:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[24895]: cluster 2024-09-13T00:38:52.278008+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e32: y(active, since 56s), standbys: x 2024-09-13T00:38:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[24895]: cluster 2024-09-13T00:38:52.279047+0000 mon.a (mon.0) 7 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-09-13T00:38:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[24895]: cluster 2024-09-13T00:38:52.279088+0000 mon.a (mon.0) 8 : cluster [WRN] [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T00:38:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[24895]: cluster 2024-09-13T00:38:52.279105+0000 mon.a (mon.0) 9 : cluster [WRN] application not enabled on pool 'test-librbd-smithi077-68798-8' 2024-09-13T00:38:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[24895]: cluster 2024-09-13T00:38:52.279120+0000 mon.a (mon.0) 10 : cluster [WRN] use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T00:38:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[24895]: cluster 2024-09-13T00:38:52.297775+0000 mon.a (mon.0) 11 : cluster [DBG] mgrmap e33: y(active, since 56s), standbys: x 2024-09-13T00:38:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:52 smithi088 bash[24895]: audit 2024-09-13T00:38:52.298039+0000 mon.a (mon.0) 12 : audit [INF] from='mgr.25064 ' entity='' 2024-09-13T00:38:52.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[73137]: ignoring --setuser ceph since I am not root 2024-09-13T00:38:52.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[73137]: ignoring --setgroup ceph since I am not root 2024-09-13T00:38:52.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[73137]: debug 2024-09-13T00:38:52.370+0000 7ffaf2ed9640 1 -- 172.21.15.77:0/2712016808 <== mon.2 v2:172.21.15.77:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561c8fb5cd00 con 0x561c8fb4f400 2024-09-13T00:38:52.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[73137]: debug 2024-09-13T00:38:52.498+0000 7ffaf5734040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-13T00:38:52.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[32040]: cluster 2024-09-13T00:38:48.279731+0000 mgr.y (mgr.25064) 49 : cluster [DBG] pgmap v77: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 324 KiB/s rd, 19 KiB/s wr, 413 op/s 2024-09-13T00:38:52.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[32040]: audit 2024-09-13T00:38:48.605678+0000 mon.c (mon.2) 233 : audit [DBG] from='client.? 172.21.15.77:0/3167492330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:52.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[32040]: cluster 2024-09-13T00:38:50.280456+0000 mgr.y (mgr.25064) 50 : cluster [DBG] pgmap v78: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 220 KiB/s rd, 13 KiB/s wr, 280 op/s 2024-09-13T00:38:52.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[32040]: audit 2024-09-13T00:38:50.970260+0000 mon.c (mon.2) 234 : audit [DBG] from='client.? 172.21.15.77:0/2141773463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:52.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[32040]: cluster 2024-09-13T00:38:51.424459+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-09-13T00:38:52.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[32040]: cluster 2024-09-13T00:38:52.248763+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T00:38:52.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[32040]: cluster 2024-09-13T00:38:52.257963+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:38:52.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[32040]: cluster 2024-09-13T00:38:52.275698+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:38:52.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[32040]: cluster 2024-09-13T00:38:52.275753+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2024-09-13T00:38:52.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[32040]: cluster 2024-09-13T00:38:52.278008+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e32: y(active, since 56s), standbys: x 2024-09-13T00:38:52.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[32040]: cluster 2024-09-13T00:38:52.279047+0000 mon.a (mon.0) 7 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-09-13T00:38:52.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[32040]: cluster 2024-09-13T00:38:52.279088+0000 mon.a (mon.0) 8 : cluster [WRN] [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T00:38:52.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[32040]: cluster 2024-09-13T00:38:52.279105+0000 mon.a (mon.0) 9 : cluster [WRN] application not enabled on pool 'test-librbd-smithi077-68798-8' 2024-09-13T00:38:52.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[32040]: cluster 2024-09-13T00:38:52.279120+0000 mon.a (mon.0) 10 : cluster [WRN] use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T00:38:52.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[32040]: cluster 2024-09-13T00:38:52.297775+0000 mon.a (mon.0) 11 : cluster [DBG] mgrmap e33: y(active, since 56s), standbys: x 2024-09-13T00:38:52.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[32040]: audit 2024-09-13T00:38:52.298039+0000 mon.a (mon.0) 12 : audit [INF] from='mgr.25064 ' entity='' 2024-09-13T00:38:52.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: debug 2024-09-13T00:38:52.326+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:38:52.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: cluster 2024-09-13T00:38:48.279731+0000 mgr.y (mgr.25064) 49 : cluster [DBG] pgmap v77: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 324 KiB/s rd, 19 KiB/s wr, 413 op/s 2024-09-13T00:38:52.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: audit 2024-09-13T00:38:48.605678+0000 mon.c (mon.2) 233 : audit [DBG] from='client.? 172.21.15.77:0/3167492330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:52.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: cluster 2024-09-13T00:38:50.280456+0000 mgr.y (mgr.25064) 50 : cluster [DBG] pgmap v78: 137 pgs: 137 active+clean; 901 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 220 KiB/s rd, 13 KiB/s wr, 280 op/s 2024-09-13T00:38:52.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: audit 2024-09-13T00:38:50.970260+0000 mon.c (mon.2) 234 : audit [DBG] from='client.? 172.21.15.77:0/2141773463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:52.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: cluster 2024-09-13T00:38:51.424459+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-09-13T00:38:52.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: cluster 2024-09-13T00:38:52.248763+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T00:38:52.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: cluster 2024-09-13T00:38:52.257963+0000 mon.a (mon.0) 3 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:38:52.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: cluster 2024-09-13T00:38:52.275698+0000 mon.a (mon.0) 4 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:38:52.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: cluster 2024-09-13T00:38:52.275753+0000 mon.a (mon.0) 5 : cluster [DBG] osdmap e282: 8 total, 8 up, 8 in 2024-09-13T00:38:52.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: cluster 2024-09-13T00:38:52.278008+0000 mon.a (mon.0) 6 : cluster [DBG] mgrmap e32: y(active, since 56s), standbys: x 2024-09-13T00:38:52.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: cluster 2024-09-13T00:38:52.279047+0000 mon.a (mon.0) 7 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-09-13T00:38:52.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: cluster 2024-09-13T00:38:52.279088+0000 mon.a (mon.0) 8 : cluster [WRN] [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T00:38:52.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: cluster 2024-09-13T00:38:52.279105+0000 mon.a (mon.0) 9 : cluster [WRN] application not enabled on pool 'test-librbd-smithi077-68798-8' 2024-09-13T00:38:52.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: cluster 2024-09-13T00:38:52.279120+0000 mon.a (mon.0) 10 : cluster [WRN] use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T00:38:52.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: cluster 2024-09-13T00:38:52.297775+0000 mon.a (mon.0) 11 : cluster [DBG] mgrmap e33: y(active, since 56s), standbys: x 2024-09-13T00:38:52.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:52 smithi077 bash[80681]: audit 2024-09-13T00:38:52.298039+0000 mon.a (mon.0) 12 : audit [INF] from='mgr.25064 ' entity='' 2024-09-13T00:38:53.322 INFO:teuthology.orchestra.run.smithi077.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 1 2024-09-13T00:38:53.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:53 smithi088 bash[24895]: cluster 2024-09-13T00:38:52.330136+0000 mon.a (mon.0) 13 : cluster [ERR] Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:38:53.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:53 smithi088 bash[24895]: cluster 2024-09-13T00:38:52.339374+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2024-09-13T00:38:53.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:53 smithi088 bash[24895]: audit 2024-09-13T00:38:53.315131+0000 mon.c (mon.2) 235 : audit [DBG] from='client.? 172.21.15.77:0/1167479032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:53.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:53 smithi088 bash[61791]: ts=2024-09-13T00:38:53.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:53.711 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:53 smithi077 bash[80681]: cluster 2024-09-13T00:38:52.330136+0000 mon.a (mon.0) 13 : cluster [ERR] Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:38:53.711 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:53 smithi077 bash[80681]: cluster 2024-09-13T00:38:52.339374+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2024-09-13T00:38:53.711 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:53 smithi077 bash[80681]: audit 2024-09-13T00:38:53.315131+0000 mon.c (mon.2) 235 : audit [DBG] from='client.? 172.21.15.77:0/1167479032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:53.712 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:53 smithi077 bash[32040]: cluster 2024-09-13T00:38:52.330136+0000 mon.a (mon.0) 13 : cluster [ERR] Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:38:53.712 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:53 smithi077 bash[32040]: cluster 2024-09-13T00:38:52.339374+0000 mon.a (mon.0) 14 : cluster [DBG] osdmap e283: 8 total, 8 up, 8 in 2024-09-13T00:38:53.712 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:53 smithi077 bash[32040]: audit 2024-09-13T00:38:53.315131+0000 mon.c (mon.2) 235 : audit [DBG] from='client.? 172.21.15.77:0/1167479032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:38:54.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:53 smithi077 bash[73137]: debug 2024-09-13T00:38:53.714+0000 7ffaf5734040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T00:38:54.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:53 smithi077 bash[73137]: debug 2024-09-13T00:38:53.778+0000 7ffaf5734040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T00:38:54.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:53 smithi077 bash[73137]: debug 2024-09-13T00:38:53.926+0000 7ffaf5734040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T00:38:54.100 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:53 smithi088 bash[62189]: debug 2024-09-13T00:38:53.681+0000 7f695fbf0040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-13T00:38:54.101 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:53 smithi088 bash[62189]: debug 2024-09-13T00:38:53.745+0000 7f695fbf0040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-13T00:38:54.101 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:53 smithi088 bash[62189]: debug 2024-09-13T00:38:53.897+0000 7f695fbf0040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-13T00:38:54.275 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph orch ps' 2024-09-13T00:38:54.382 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap1 2024-09-13T00:38:54.496 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:38:54.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:54 smithi077 bash[80681]: cluster 2024-09-13T00:38:53.351204+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2024-09-13T00:38:54.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:54 smithi077 bash[32040]: cluster 2024-09-13T00:38:53.351204+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2024-09-13T00:38:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:54 smithi088 bash[24895]: cluster 2024-09-13T00:38:53.351204+0000 mon.a (mon.0) 15 : cluster [DBG] osdmap e284: 8 total, 8 up, 8 in 2024-09-13T00:38:54.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:54 smithi088 bash[62189]: debug 2024-09-13T00:38:54.565+0000 7f695fbf0040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T00:38:54.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:54 smithi088 bash[61791]: ts=2024-09-13T00:38:54.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:54.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:54 smithi088 bash[61791]: ts=2024-09-13T00:38:54.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:54.913 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:54 smithi077 bash[73137]: debug 2024-09-13T00:38:54.598+0000 7ffaf5734040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-13T00:38:54.913 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:54 smithi077 bash[73137]: debug 2024-09-13T00:38:54.666+0000 7ffaf5734040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T00:38:54.913 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:54 smithi077 bash[73137]: debug 2024-09-13T00:38:54.734+0000 7ffaf5734040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T00:38:54.963 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:54 smithi088 bash[62189]: debug 2024-09-13T00:38:54.641+0000 7f695fbf0040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-13T00:38:54.963 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:54 smithi088 bash[62189]: debug 2024-09-13T00:38:54.709+0000 7f695fbf0040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-13T00:38:54.963 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:54 smithi088 bash[62189]: debug 2024-09-13T00:38:54.889+0000 7f695fbf0040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T00:38:54.964 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:54 smithi088 bash[62189]: debug 2024-09-13T00:38:54.965+0000 7f695fbf0040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T00:38:55.277 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:54 smithi077 bash[73137]: debug 2024-09-13T00:38:54.914+0000 7ffaf5734040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-13T00:38:55.277 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:54 smithi077 bash[73137]: debug 2024-09-13T00:38:54.990+0000 7ffaf5734040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-13T00:38:55.277 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:55 smithi077 bash[73137]: debug 2024-09-13T00:38:55.058+0000 7ffaf5734040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T00:38:55.287 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:55 smithi088 bash[62189]: debug 2024-09-13T00:38:55.029+0000 7f695fbf0040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-13T00:38:55.572 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:55 smithi088 bash[62189]: debug 2024-09-13T00:38:55.289+0000 7f695fbf0040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T00:38:55.572 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:55 smithi088 bash[62189]: debug 2024-09-13T00:38:55.357+0000 7f695fbf0040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T00:38:55.572 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:55 smithi088 bash[24895]: cluster 2024-09-13T00:38:54.366515+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2024-09-13T00:38:55.604 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:55 smithi077 bash[80681]: cluster 2024-09-13T00:38:54.366515+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2024-09-13T00:38:55.605 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:55 smithi077 bash[73137]: debug 2024-09-13T00:38:55.314+0000 7ffaf5734040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-13T00:38:55.605 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:55 smithi077 bash[73137]: debug 2024-09-13T00:38:55.394+0000 7ffaf5734040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-13T00:38:55.605 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:55 smithi077 bash[32040]: cluster 2024-09-13T00:38:54.366515+0000 mon.a (mon.0) 16 : cluster [DBG] osdmap e285: 8 total, 8 up, 8 in 2024-09-13T00:38:55.750 INFO:tasks.workunit.client.0.smithi077.stdout:migration_abort 2024-09-13T00:38:55.851 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:55 smithi088 bash[62189]: debug 2024-09-13T00:38:55.573+0000 7f695fbf0040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T00:38:55.851 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:55 smithi088 bash[62189]: debug 2024-09-13T00:38:55.653+0000 7f695fbf0040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T00:38:55.970 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:55 smithi077 bash[73137]: debug 2024-09-13T00:38:55.606+0000 7ffaf5734040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-13T00:38:55.970 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:55 smithi077 bash[73137]: debug 2024-09-13T00:38:55.694+0000 7ffaf5734040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-13T00:38:56.261 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:55 smithi088 bash[62189]: debug 2024-09-13T00:38:55.929+0000 7f695fbf0040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T00:38:56.261 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:56 smithi088 bash[62189]: debug 2024-09-13T00:38:56.101+0000 7f695fbf0040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:38:56.277 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:55 smithi077 bash[73137]: debug 2024-09-13T00:38:55.970+0000 7ffaf5734040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-13T00:38:56.277 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:56 smithi077 bash[73137]: debug 2024-09-13T00:38:56.146+0000 7ffaf5734040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:38:56.513 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:56 smithi088 bash[24895]: cluster 2024-09-13T00:38:55.372428+0000 mon.a (mon.0) 17 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2024-09-13T00:38:56.514 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:56 smithi088 bash[24895]: cluster 2024-09-13T00:38:55.821606+0000 mon.a (mon.0) 18 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2024-09-13T00:38:56.514 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:56 smithi088 bash[62189]: debug 2024-09-13T00:38:56.261+0000 7f695fbf0040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T00:38:56.514 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:56 smithi088 bash[62189]: debug 2024-09-13T00:38:56.337+0000 7f695fbf0040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T00:38:56.562 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:56 smithi077 bash[73137]: debug 2024-09-13T00:38:56.306+0000 7ffaf5734040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-13T00:38:56.562 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:56 smithi077 bash[73137]: debug 2024-09-13T00:38:56.386+0000 7ffaf5734040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-13T00:38:56.562 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:56 smithi077 bash[32040]: cluster 2024-09-13T00:38:55.372428+0000 mon.a (mon.0) 17 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2024-09-13T00:38:56.562 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:56 smithi077 bash[32040]: cluster 2024-09-13T00:38:55.821606+0000 mon.a (mon.0) 18 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2024-09-13T00:38:56.562 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:56 smithi077 bash[80681]: cluster 2024-09-13T00:38:55.372428+0000 mon.a (mon.0) 17 : cluster [DBG] osdmap e286: 8 total, 8 up, 8 in 2024-09-13T00:38:56.563 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:56 smithi077 bash[80681]: cluster 2024-09-13T00:38:55.821606+0000 mon.a (mon.0) 18 : cluster [DBG] osdmap e287: 8 total, 8 up, 8 in 2024-09-13T00:38:56.850 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:56 smithi088 bash[62189]: debug 2024-09-13T00:38:56.513+0000 7f695fbf0040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:38:57.026 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:56 smithi077 bash[73137]: debug 2024-09-13T00:38:56.562+0000 7ffaf5734040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-13T00:38:57.028 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.CloneV1ParentAbortRelinkNotNeeded (4550 ms) 2024-09-13T00:38:57.029 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach 2024-09-13T00:38:57.071 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:38:57.088 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 0~10 2024-09-13T00:38:57.098 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:38:57.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:57 smithi088 bash[62189]: debug 2024-09-13T00:38:57.349+0000 7f695fbf0040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T00:38:57.601 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:57 smithi088 bash[62189]: debug 2024-09-13T00:38:57.421+0000 7f695fbf0040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T00:38:57.724 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:57 smithi077 bash[73137]: debug 2024-09-13T00:38:57.430+0000 7ffaf5734040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-13T00:38:57.724 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:57 smithi077 bash[73137]: debug 2024-09-13T00:38:57.502+0000 7ffaf5734040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-13T00:38:58.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:57 smithi077 bash[73137]: debug 2024-09-13T00:38:57.726+0000 7ffaf5734040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T00:38:58.027 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:57 smithi077 bash[73137]: debug 2024-09-13T00:38:57.798+0000 7ffaf5734040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T00:38:58.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:57 smithi077 bash[32040]: cluster 2024-09-13T00:38:56.824193+0000 mon.a (mon.0) 19 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2024-09-13T00:38:58.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:57 smithi077 bash[80681]: cluster 2024-09-13T00:38:56.824193+0000 mon.a (mon.0) 19 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2024-09-13T00:38:58.099 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:57 smithi088 bash[24895]: cluster 2024-09-13T00:38:56.824193+0000 mon.a (mon.0) 19 : cluster [DBG] osdmap e288: 8 total, 8 up, 8 in 2024-09-13T00:38:58.099 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:57 smithi088 bash[62189]: debug 2024-09-13T00:38:57.641+0000 7f695fbf0040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-13T00:38:58.099 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:57 smithi088 bash[62189]: debug 2024-09-13T00:38:57.713+0000 7f695fbf0040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-13T00:38:58.319 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap1 2024-09-13T00:38:58.351 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[62189]: debug 2024-09-13T00:38:58.101+0000 7f695fbf0040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T00:38:58.351 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[62189]: debug 2024-09-13T00:38:58.177+0000 7f695fbf0040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T00:38:58.351 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[62189]: [13/Sep/2024:00:38:58] ENGINE Bus STARTING 2024-09-13T00:38:58.351 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[62189]: CherryPy Checker: 2024-09-13T00:38:58.351 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[62189]: The Application mounted at '' has an empty config. 2024-09-13T00:38:58.351 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[62189]: [13/Sep/2024:00:38:58] ENGINE Serving on http://:::9283 2024-09-13T00:38:58.351 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[62189]: [13/Sep/2024:00:38:58] ENGINE Bus STARTED 2024-09-13T00:38:58.378 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:38:58.477 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: debug 2024-09-13T00:38:58.186+0000 7ffaf5734040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-13T00:38:58.477 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: debug 2024-09-13T00:38:58.266+0000 7ffaf5734040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-13T00:38:58.477 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: [13/Sep/2024:00:38:58] ENGINE Bus STARTING 2024-09-13T00:38:58.478 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: CherryPy Checker: 2024-09-13T00:38:58.478 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: The Application mounted at '' has an empty config. 2024-09-13T00:38:58.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: debug 2024-09-13T00:38:58.478+0000 7ffa5ca31640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:38:58.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: debug 2024-09-13T00:38:58.478+0000 7ffa5ca31640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:38:58.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: debug 2024-09-13T00:38:58.478+0000 7ffa5ca31640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:38:58.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: debug 2024-09-13T00:38:58.478+0000 7ffa5ca31640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:38:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: debug 2024-09-13T00:38:58.478+0000 7ffa5ca31640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:38:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: debug 2024-09-13T00:38:58.482+0000 7ffa5fa37640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:38:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: debug 2024-09-13T00:38:58.482+0000 7ffa5fa37640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:38:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: debug 2024-09-13T00:38:58.482+0000 7ffa5fa37640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:38:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: debug 2024-09-13T00:38:58.482+0000 7ffa5fa37640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:38:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: debug 2024-09-13T00:38:58.482+0000 7ffa5fa37640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-13T00:38:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:38:58] "GET /metrics HTTP/1.1" 200 36041 "" "Prometheus/2.43.0" 2024-09-13T00:38:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: [13/Sep/2024:00:38:58] ENGINE Serving on http://:::9283 2024-09-13T00:38:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[73137]: [13/Sep/2024:00:38:58] ENGINE Bus STARTED 2024-09-13T00:38:58.830 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[61791]: ts=2024-09-13T00:38:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:38:59.041 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:38:59.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: cluster 2024-09-13T00:38:57.826275+0000 mon.a (mon.0) 20 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2024-09-13T00:38:59.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: cluster 2024-09-13T00:38:58.202448+0000 mon.a (mon.0) 21 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T00:38:59.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: cluster 2024-09-13T00:38:58.202577+0000 mon.a (mon.0) 22 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:38:59.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.205004+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.? 172.21.15.88:0/1215191939' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:38:59.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.205988+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.? 172.21.15.88:0/1215191939' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:38:59.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.207957+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.? 172.21.15.88:0/1215191939' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:38:59.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.209381+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.? 172.21.15.88:0/1215191939' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:38:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: cluster 2024-09-13T00:38:58.290130+0000 mon.a (mon.0) 23 : cluster [INF] Active manager daemon y restarted 2024-09-13T00:38:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: cluster 2024-09-13T00:38:58.290903+0000 mon.a (mon.0) 24 : cluster [INF] Activating manager daemon y 2024-09-13T00:38:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: cluster 2024-09-13T00:38:58.304951+0000 mon.a (mon.0) 25 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2024-09-13T00:38:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: cluster 2024-09-13T00:38:58.305852+0000 mon.a (mon.0) 26 : cluster [DBG] mgrmap e34: y(active, starting, since 0.0150911s), standbys: x 2024-09-13T00:38:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.313991+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:38:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.314159+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:38:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.314399+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:38:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.315363+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi077.bigbbf"}]: dispatch 2024-09-13T00:38:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.315618+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi088.krmowh"}]: dispatch 2024-09-13T00:38:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.315947+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T00:38:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.316103+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T00:38:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.316258+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:38:59.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.316440+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:38:59.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.316633+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:38:59.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.316828+0000 mon.b (mon.1) 105 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:38:59.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.317132+0000 mon.b (mon.1) 106 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:38:59.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.317415+0000 mon.b (mon.1) 107 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:38:59.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.317692+0000 mon.b (mon.1) 108 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:38:59.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.318023+0000 mon.b (mon.1) 109 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:38:59.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.318377+0000 mon.b (mon.1) 110 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T00:38:59.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.318641+0000 mon.b (mon.1) 111 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T00:38:59.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.319042+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T00:38:59.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: cluster 2024-09-13T00:38:58.349509+0000 mon.a (mon.0) 27 : cluster [INF] Manager daemon y is now available 2024-09-13T00:38:59.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.373934+0000 mon.b (mon.1) 113 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:38:59.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.409751+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:38:59.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.411389+0000 mon.b (mon.1) 114 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:59.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.438494+0000 mon.b (mon.1) 115 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:38:59.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.439878+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.25304 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:38:59.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.458732+0000 mon.b (mon.1) 116 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:38:59.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:38:58 smithi088 bash[24895]: audit 2024-09-13T00:38:58.460200+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.25304 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:38:59.169 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: cluster 2024-09-13T00:38:57.826275+0000 mon.a (mon.0) 20 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2024-09-13T00:38:59.169 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: cluster 2024-09-13T00:38:58.202448+0000 mon.a (mon.0) 21 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T00:38:59.169 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: cluster 2024-09-13T00:38:58.202577+0000 mon.a (mon.0) 22 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:38:59.170 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.205004+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.? 172.21.15.88:0/1215191939' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:38:59.170 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.205988+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.? 172.21.15.88:0/1215191939' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:38:59.170 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.207957+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.? 172.21.15.88:0/1215191939' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:38:59.170 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.209381+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.? 172.21.15.88:0/1215191939' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:38:59.170 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: cluster 2024-09-13T00:38:58.290130+0000 mon.a (mon.0) 23 : cluster [INF] Active manager daemon y restarted 2024-09-13T00:38:59.170 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: cluster 2024-09-13T00:38:58.290903+0000 mon.a (mon.0) 24 : cluster [INF] Activating manager daemon y 2024-09-13T00:38:59.170 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: cluster 2024-09-13T00:38:58.304951+0000 mon.a (mon.0) 25 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2024-09-13T00:38:59.170 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: cluster 2024-09-13T00:38:58.305852+0000 mon.a (mon.0) 26 : cluster [DBG] mgrmap e34: y(active, starting, since 0.0150911s), standbys: x 2024-09-13T00:38:59.170 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.313991+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:38:59.170 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.314159+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:38:59.170 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.314399+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:38:59.170 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.315363+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi077.bigbbf"}]: dispatch 2024-09-13T00:38:59.170 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.315618+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi088.krmowh"}]: dispatch 2024-09-13T00:38:59.171 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.315947+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T00:38:59.171 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.316103+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T00:38:59.171 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.316258+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:38:59.171 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.316440+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:38:59.171 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.316633+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:38:59.171 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.316828+0000 mon.b (mon.1) 105 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:38:59.171 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.317132+0000 mon.b (mon.1) 106 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:38:59.171 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.317415+0000 mon.b (mon.1) 107 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:38:59.171 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.317692+0000 mon.b (mon.1) 108 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:38:59.171 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.318023+0000 mon.b (mon.1) 109 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:38:59.171 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.318377+0000 mon.b (mon.1) 110 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T00:38:59.171 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.318641+0000 mon.b (mon.1) 111 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T00:38:59.171 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.319042+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T00:38:59.172 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: cluster 2024-09-13T00:38:58.349509+0000 mon.a (mon.0) 27 : cluster [INF] Manager daemon y is now available 2024-09-13T00:38:59.172 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.373934+0000 mon.b (mon.1) 113 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:38:59.172 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.409751+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:38:59.172 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.411389+0000 mon.b (mon.1) 114 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:59.172 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.438494+0000 mon.b (mon.1) 115 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:38:59.172 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.439878+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.25304 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:38:59.172 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.458732+0000 mon.b (mon.1) 116 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:38:59.172 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[32040]: audit 2024-09-13T00:38:58.460200+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.25304 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:38:59.173 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: cluster 2024-09-13T00:38:57.826275+0000 mon.a (mon.0) 20 : cluster [DBG] osdmap e289: 8 total, 8 up, 8 in 2024-09-13T00:38:59.173 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: cluster 2024-09-13T00:38:58.202448+0000 mon.a (mon.0) 21 : cluster [DBG] Standby manager daemon x restarted 2024-09-13T00:38:59.173 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: cluster 2024-09-13T00:38:58.202577+0000 mon.a (mon.0) 22 : cluster [DBG] Standby manager daemon x started 2024-09-13T00:38:59.173 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.205004+0000 mon.b (mon.1) 91 : audit [DBG] from='mgr.? 172.21.15.88:0/1215191939' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-13T00:38:59.173 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.205988+0000 mon.b (mon.1) 92 : audit [DBG] from='mgr.? 172.21.15.88:0/1215191939' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-13T00:38:59.173 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.207957+0000 mon.b (mon.1) 93 : audit [DBG] from='mgr.? 172.21.15.88:0/1215191939' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-13T00:38:59.173 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.209381+0000 mon.b (mon.1) 94 : audit [DBG] from='mgr.? 172.21.15.88:0/1215191939' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-13T00:38:59.173 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: cluster 2024-09-13T00:38:58.290130+0000 mon.a (mon.0) 23 : cluster [INF] Active manager daemon y restarted 2024-09-13T00:38:59.173 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: cluster 2024-09-13T00:38:58.290903+0000 mon.a (mon.0) 24 : cluster [INF] Activating manager daemon y 2024-09-13T00:38:59.173 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: cluster 2024-09-13T00:38:58.304951+0000 mon.a (mon.0) 25 : cluster [DBG] osdmap e290: 8 total, 8 up, 8 in 2024-09-13T00:38:59.173 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: cluster 2024-09-13T00:38:58.305852+0000 mon.a (mon.0) 26 : cluster [DBG] mgrmap e34: y(active, starting, since 0.0150911s), standbys: x 2024-09-13T00:38:59.174 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.313991+0000 mon.b (mon.1) 95 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:38:59.174 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.314159+0000 mon.b (mon.1) 96 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:38:59.174 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.314399+0000 mon.b (mon.1) 97 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:38:59.174 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.315363+0000 mon.b (mon.1) 98 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi077.bigbbf"}]: dispatch 2024-09-13T00:38:59.174 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.315618+0000 mon.b (mon.1) 99 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi088.krmowh"}]: dispatch 2024-09-13T00:38:59.174 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.315947+0000 mon.b (mon.1) 100 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-13T00:38:59.174 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.316103+0000 mon.b (mon.1) 101 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-13T00:38:59.174 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.316258+0000 mon.b (mon.1) 102 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:38:59.174 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.316440+0000 mon.b (mon.1) 103 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:38:59.174 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.316633+0000 mon.b (mon.1) 104 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:38:59.174 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.316828+0000 mon.b (mon.1) 105 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:38:59.174 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.317132+0000 mon.b (mon.1) 106 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:38:59.174 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.317415+0000 mon.b (mon.1) 107 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-13T00:38:59.175 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.317692+0000 mon.b (mon.1) 108 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-13T00:38:59.175 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.318023+0000 mon.b (mon.1) 109 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-13T00:38:59.175 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.318377+0000 mon.b (mon.1) 110 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-13T00:38:59.175 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.318641+0000 mon.b (mon.1) 111 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-13T00:38:59.175 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.319042+0000 mon.b (mon.1) 112 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-13T00:38:59.175 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: cluster 2024-09-13T00:38:58.349509+0000 mon.a (mon.0) 27 : cluster [INF] Manager daemon y is now available 2024-09-13T00:38:59.175 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.373934+0000 mon.b (mon.1) 113 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:38:59.175 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.409751+0000 mon.a (mon.0) 28 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:38:59.175 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.411389+0000 mon.b (mon.1) 114 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:38:59.175 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.438494+0000 mon.b (mon.1) 115 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:38:59.175 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.439878+0000 mon.a (mon.0) 29 : audit [INF] from='mgr.25304 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-13T00:38:59.175 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.458732+0000 mon.b (mon.1) 116 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:38:59.175 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:38:58 smithi077 bash[80681]: audit 2024-09-13T00:38:58.460200+0000 mon.a (mon.0) 30 : audit [INF] from='mgr.25304 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-13T00:38:59.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:38:59 smithi077 bash[73137]: debug 2024-09-13T00:38:59.310+0000 7ffa94da1640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-13T00:38:59.761 INFO:tasks.workunit.client.0.smithi077.stdout:migration_abort 2024-09-13T00:39:00.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[80681]: cluster 2024-09-13T00:38:59.308201+0000 mon.a (mon.0) 31 : cluster [DBG] mgrmap e35: y(active, since 1.01746s), standbys: x 2024-09-13T00:39:00.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[80681]: cluster 2024-09-13T00:38:59.327241+0000 mgr.y (mgr.25304) 1 : cluster [DBG] pgmap v3: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:39:00.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[80681]: cluster 2024-09-13T00:38:59.413824+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2024-09-13T00:39:00.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[32040]: cluster 2024-09-13T00:38:59.308201+0000 mon.a (mon.0) 31 : cluster [DBG] mgrmap e35: y(active, since 1.01746s), standbys: x 2024-09-13T00:39:00.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[32040]: cluster 2024-09-13T00:38:59.327241+0000 mgr.y (mgr.25304) 1 : cluster [DBG] pgmap v3: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:39:00.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[32040]: cluster 2024-09-13T00:38:59.413824+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2024-09-13T00:39:00.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: debug 2024-09-13T00:39:00.070+0000 7ffa43c40640 -1 log_channel(cephadm) log [ERR] : [13/Sep/2024:00:39:00] ENGINE Error in HTTPServer.serve 2024-09-13T00:39:00.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: Traceback (most recent call last): 2024-09-13T00:39:00.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:39:00.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: self._connections.run(self.expiration_interval) 2024-09-13T00:39:00.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:39:00.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: self._run(expiration_interval) 2024-09-13T00:39:00.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:39:00.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:39:00.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:39:00.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:39:00.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:39:00.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: s = self.context.wrap_socket( 2024-09-13T00:39:00.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:39:00.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: return self.sslsocket_class._create( 2024-09-13T00:39:00.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:39:00.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: self.do_handshake() 2024-09-13T00:39:00.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:39:00.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: self._sslobj.do_handshake() 2024-09-13T00:39:00.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:00 smithi077 bash[73137]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:39:00.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:00 smithi088 bash[24895]: cluster 2024-09-13T00:38:59.308201+0000 mon.a (mon.0) 31 : cluster [DBG] mgrmap e35: y(active, since 1.01746s), standbys: x 2024-09-13T00:39:00.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:00 smithi088 bash[24895]: cluster 2024-09-13T00:38:59.327241+0000 mgr.y (mgr.25304) 1 : cluster [DBG] pgmap v3: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:39:00.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:00 smithi088 bash[24895]: cluster 2024-09-13T00:38:59.413824+0000 mon.a (mon.0) 32 : cluster [DBG] osdmap e291: 8 total, 8 up, 8 in 2024-09-13T00:39:00.648 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach (3620 ms) 2024-09-13T00:39:00.649 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixParentReattach 2024-09-13T00:39:00.719 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:39:00.740 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 0~10 2024-09-13T00:39:00.755 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:39:01.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: cephadm 2024-09-13T00:38:59.844428+0000 mgr.y (mgr.25304) 2 : cephadm [INF] [13/Sep/2024:00:38:59] ENGINE Bus STARTING 2024-09-13T00:39:01.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: cephadm 2024-09-13T00:38:59.946509+0000 mgr.y (mgr.25304) 3 : cephadm [INF] [13/Sep/2024:00:38:59] ENGINE Serving on http://172.21.15.77:8765 2024-09-13T00:39:01.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: cephadm 2024-09-13T00:39:00.068215+0000 mgr.y (mgr.25304) 4 : cephadm [INF] [13/Sep/2024:00:39:00] ENGINE Serving on https://172.21.15.77:7150 2024-09-13T00:39:01.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: cephadm 2024-09-13T00:39:00.068359+0000 mgr.y (mgr.25304) 5 : cephadm [INF] [13/Sep/2024:00:39:00] ENGINE Bus STARTED 2024-09-13T00:39:01.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: cephadm 2024-09-13T00:39:00.070636+0000 mgr.y (mgr.25304) 6 : cephadm [ERR] [13/Sep/2024:00:39:00] ENGINE Error in HTTPServer.serve 2024-09-13T00:39:01.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: Traceback (most recent call last): 2024-09-13T00:39:01.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:39:01.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: self._connections.run(self.expiration_interval) 2024-09-13T00:39:01.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:39:01.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: self._run(expiration_interval) 2024-09-13T00:39:01.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:39:01.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:39:01.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:39:01.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:39:01.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:39:01.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: s = self.context.wrap_socket( 2024-09-13T00:39:01.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:39:01.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: return self.sslsocket_class._create( 2024-09-13T00:39:01.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:39:01.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: self.do_handshake() 2024-09-13T00:39:01.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:39:01.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: self._sslobj.do_handshake() 2024-09-13T00:39:01.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:39:01.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: cluster 2024-09-13T00:39:00.319326+0000 mgr.y (mgr.25304) 7 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:39:01.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: cluster 2024-09-13T00:39:00.421862+0000 mon.a (mon.0) 33 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2024-09-13T00:39:01.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:01 smithi088 bash[24895]: cluster 2024-09-13T00:39:00.820274+0000 mon.a (mon.0) 34 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2024-09-13T00:39:01.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: cephadm 2024-09-13T00:38:59.844428+0000 mgr.y (mgr.25304) 2 : cephadm [INF] [13/Sep/2024:00:38:59] ENGINE Bus STARTING 2024-09-13T00:39:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: cephadm 2024-09-13T00:38:59.946509+0000 mgr.y (mgr.25304) 3 : cephadm [INF] [13/Sep/2024:00:38:59] ENGINE Serving on http://172.21.15.77:8765 2024-09-13T00:39:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: cephadm 2024-09-13T00:39:00.068215+0000 mgr.y (mgr.25304) 4 : cephadm [INF] [13/Sep/2024:00:39:00] ENGINE Serving on https://172.21.15.77:7150 2024-09-13T00:39:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: cephadm 2024-09-13T00:39:00.068359+0000 mgr.y (mgr.25304) 5 : cephadm [INF] [13/Sep/2024:00:39:00] ENGINE Bus STARTED 2024-09-13T00:39:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: cephadm 2024-09-13T00:39:00.070636+0000 mgr.y (mgr.25304) 6 : cephadm [ERR] [13/Sep/2024:00:39:00] ENGINE Error in HTTPServer.serve 2024-09-13T00:39:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: Traceback (most recent call last): 2024-09-13T00:39:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:39:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: self._connections.run(self.expiration_interval) 2024-09-13T00:39:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:39:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: self._run(expiration_interval) 2024-09-13T00:39:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:39:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:39:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:39:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:39:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:39:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: s = self.context.wrap_socket( 2024-09-13T00:39:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:39:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: return self.sslsocket_class._create( 2024-09-13T00:39:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:39:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: self.do_handshake() 2024-09-13T00:39:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:39:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: self._sslobj.do_handshake() 2024-09-13T00:39:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:39:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: cluster 2024-09-13T00:39:00.319326+0000 mgr.y (mgr.25304) 7 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:39:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: cluster 2024-09-13T00:39:00.421862+0000 mon.a (mon.0) 33 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2024-09-13T00:39:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[80681]: cluster 2024-09-13T00:39:00.820274+0000 mon.a (mon.0) 34 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2024-09-13T00:39:01.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: cephadm 2024-09-13T00:38:59.844428+0000 mgr.y (mgr.25304) 2 : cephadm [INF] [13/Sep/2024:00:38:59] ENGINE Bus STARTING 2024-09-13T00:39:01.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: cephadm 2024-09-13T00:38:59.946509+0000 mgr.y (mgr.25304) 3 : cephadm [INF] [13/Sep/2024:00:38:59] ENGINE Serving on http://172.21.15.77:8765 2024-09-13T00:39:01.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: cephadm 2024-09-13T00:39:00.068215+0000 mgr.y (mgr.25304) 4 : cephadm [INF] [13/Sep/2024:00:39:00] ENGINE Serving on https://172.21.15.77:7150 2024-09-13T00:39:01.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: cephadm 2024-09-13T00:39:00.068359+0000 mgr.y (mgr.25304) 5 : cephadm [INF] [13/Sep/2024:00:39:00] ENGINE Bus STARTED 2024-09-13T00:39:01.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: cephadm 2024-09-13T00:39:00.070636+0000 mgr.y (mgr.25304) 6 : cephadm [ERR] [13/Sep/2024:00:39:00] ENGINE Error in HTTPServer.serve 2024-09-13T00:39:01.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: Traceback (most recent call last): 2024-09-13T00:39:01.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-13T00:39:01.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: self._connections.run(self.expiration_interval) 2024-09-13T00:39:01.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-13T00:39:01.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: self._run(expiration_interval) 2024-09-13T00:39:01.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-13T00:39:01.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: new_conn = self._from_server_socket(self.server.socket) 2024-09-13T00:39:01.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-13T00:39:01.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-13T00:39:01.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-13T00:39:01.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: s = self.context.wrap_socket( 2024-09-13T00:39:01.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-13T00:39:01.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: return self.sslsocket_class._create( 2024-09-13T00:39:01.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-13T00:39:01.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: self.do_handshake() 2024-09-13T00:39:01.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-13T00:39:01.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: self._sslobj.do_handshake() 2024-09-13T00:39:01.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-13T00:39:01.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: cluster 2024-09-13T00:39:00.319326+0000 mgr.y (mgr.25304) 7 : cluster [DBG] pgmap v5: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:39:01.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: cluster 2024-09-13T00:39:00.421862+0000 mon.a (mon.0) 33 : cluster [DBG] osdmap e292: 8 total, 8 up, 8 in 2024-09-13T00:39:01.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:01 smithi077 bash[32040]: cluster 2024-09-13T00:39:00.820274+0000 mon.a (mon.0) 34 : cluster [DBG] osdmap e293: 8 total, 8 up, 8 in 2024-09-13T00:39:01.844 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap1 2024-09-13T00:39:01.990 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:39:02.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:02 smithi088 bash[24895]: cluster 2024-09-13T00:39:01.335925+0000 mon.a (mon.0) 35 : cluster [DBG] mgrmap e36: y(active, since 3s), standbys: x 2024-09-13T00:39:02.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:02 smithi088 bash[24895]: cluster 2024-09-13T00:39:01.827876+0000 mon.a (mon.0) 36 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2024-09-13T00:39:02.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:02 smithi088 bash[61791]: ts=2024-09-13T00:39:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:02.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:02 smithi077 bash[80681]: cluster 2024-09-13T00:39:01.335925+0000 mon.a (mon.0) 35 : cluster [DBG] mgrmap e36: y(active, since 3s), standbys: x 2024-09-13T00:39:02.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:02 smithi077 bash[80681]: cluster 2024-09-13T00:39:01.827876+0000 mon.a (mon.0) 36 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2024-09-13T00:39:02.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:02 smithi077 bash[32040]: cluster 2024-09-13T00:39:01.335925+0000 mon.a (mon.0) 35 : cluster [DBG] mgrmap e36: y(active, since 3s), standbys: x 2024-09-13T00:39:02.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:02 smithi077 bash[32040]: cluster 2024-09-13T00:39:01.827876+0000 mon.a (mon.0) 36 : cluster [DBG] osdmap e294: 8 total, 8 up, 8 in 2024-09-13T00:39:03.135 INFO:tasks.workunit.client.0.smithi077.stdout:migration_abort 2024-09-13T00:39:03.275 INFO:teuthology.orchestra.run.smithi077.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T00:39:03.275 INFO:teuthology.orchestra.run.smithi077.stdout:alertmanager.a smithi077 *:9093,9094 running (104s) 60s ago 9m 14.5M - 0.25.0 c8568f914cd2 49a4b5607300 2024-09-13T00:39:03.275 INFO:teuthology.orchestra.run.smithi077.stdout:grafana.a smithi088 *:3000 running (92s) 31s ago 8m 74.0M - 9.4.7 954c08fa6188 c369fdf58d31 2024-09-13T00:39:03.275 INFO:teuthology.orchestra.run.smithi077.stdout:mds.foo.smithi077.bigbbf smithi077 running (6m) 60s ago 6m 12.9M - 17.2.7-1627-gbbdadfa3 214985049bdb 30b25ab8f2dd 2024-09-13T00:39:03.276 INFO:teuthology.orchestra.run.smithi077.stdout:mds.foo.smithi088.krmowh smithi088 running (6m) 31s ago 6m 15.7M - 17.2.7-1627-gbbdadfa3 214985049bdb ebef6ef73335 2024-09-13T00:39:03.276 INFO:teuthology.orchestra.run.smithi077.stdout:mgr.x smithi088 *:8443,9283,8765 running (37s) 31s ago 17m 128M - 18.2.4-837-g7a853b17 f21ba1eea74e fdfc79d860d1 2024-09-13T00:39:03.276 INFO:teuthology.orchestra.run.smithi077.stdout:mgr.y smithi077 *:8443,9283,8765 running (85s) 60s ago 19m 458M - 18.2.4-837-g7a853b17 f21ba1eea74e d46e77b45675 2024-09-13T00:39:03.276 INFO:teuthology.orchestra.run.smithi077.stdout:mon.a smithi077 running (19m) 60s ago 20m 60.7M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 893f92df3327 2024-09-13T00:39:03.276 INFO:teuthology.orchestra.run.smithi077.stdout:mon.b smithi088 running (18m) 31s ago 18m 47.4M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb f980dbb0ec7c 2024-09-13T00:39:03.276 INFO:teuthology.orchestra.run.smithi077.stdout:mon.c smithi077 running (17m) 60s ago 17m 46.6M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb 875f78edad39 2024-09-13T00:39:03.276 INFO:teuthology.orchestra.run.smithi077.stdout:node-exporter.a smithi077 *:9100 running (111s) 60s ago 9m 7640k - 1.5.0 0da6a335fe13 2aa0a10501f7 2024-09-13T00:39:03.276 INFO:teuthology.orchestra.run.smithi077.stdout:node-exporter.b smithi088 *:9100 running (97s) 31s ago 9m 7436k - 1.5.0 0da6a335fe13 54d4c3d89eab 2024-09-13T00:39:03.276 INFO:teuthology.orchestra.run.smithi077.stdout:osd.0 smithi077 running (16m) 60s ago 16m 97.6M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb a29e831be9aa 2024-09-13T00:39:03.276 INFO:teuthology.orchestra.run.smithi077.stdout:osd.1 smithi077 running (15m) 60s ago 15m 99.0M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 8590b3b4d2e9 2024-09-13T00:39:03.276 INFO:teuthology.orchestra.run.smithi077.stdout:osd.2 smithi077 running (14m) 60s ago 14m 122M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb b2ecae227e58 2024-09-13T00:39:03.276 INFO:teuthology.orchestra.run.smithi077.stdout:osd.3 smithi077 running (13m) 60s ago 14m 104M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb d8c3c25f4ebd 2024-09-13T00:39:03.277 INFO:teuthology.orchestra.run.smithi077.stdout:osd.4 smithi088 running (13m) 31s ago 13m 99.5M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 2c9ea6919060 2024-09-13T00:39:03.277 INFO:teuthology.orchestra.run.smithi077.stdout:osd.5 smithi088 running (12m) 31s ago 12m 107M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb e89a6b298b28 2024-09-13T00:39:03.277 INFO:teuthology.orchestra.run.smithi077.stdout:osd.6 smithi088 running (11m) 31s ago 11m 115M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 599c9665873c 2024-09-13T00:39:03.277 INFO:teuthology.orchestra.run.smithi077.stdout:osd.7 smithi088 running (10m) 31s ago 10m 141M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 0ff0388ffa3c 2024-09-13T00:39:03.277 INFO:teuthology.orchestra.run.smithi077.stdout:prometheus.a smithi088 *:9095 running (43s) 31s ago 9m 42.1M - 2.43.0 a07b618ecd1d 5fb440283869 2024-09-13T00:39:03.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:03 smithi088 bash[24895]: cluster 2024-09-13T00:39:02.320112+0000 mgr.y (mgr.25304) 8 : cluster [DBG] pgmap v9: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:39:03.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:03 smithi088 bash[24895]: cluster 2024-09-13T00:39:02.838233+0000 mon.a (mon.0) 37 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2024-09-13T00:39:03.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:03 smithi088 bash[61791]: ts=2024-09-13T00:39:03.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:03.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:03 smithi077 bash[80681]: cluster 2024-09-13T00:39:02.320112+0000 mgr.y (mgr.25304) 8 : cluster [DBG] pgmap v9: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:39:03.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:03 smithi077 bash[80681]: cluster 2024-09-13T00:39:02.838233+0000 mon.a (mon.0) 37 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2024-09-13T00:39:03.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:03 smithi077 bash[32040]: cluster 2024-09-13T00:39:02.320112+0000 mgr.y (mgr.25304) 8 : cluster [DBG] pgmap v9: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:39:03.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:03 smithi077 bash[32040]: cluster 2024-09-13T00:39:02.838233+0000 mon.a (mon.0) 37 : cluster [DBG] osdmap e295: 8 total, 8 up, 8 in 2024-09-13T00:39:03.948 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixParentReattach (3299 ms) 2024-09-13T00:39:03.948 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.CloneV2ParentAbortRelinkNotNeeded 2024-09-13T00:39:03.995 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:39:04.008 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 0~10 2024-09-13T00:39:04.019 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:39:04.112 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph orch upgrade pause' 2024-09-13T00:39:04.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:04 smithi077 bash[80681]: audit 2024-09-13T00:39:03.266194+0000 mgr.y (mgr.25304) 9 : audit [DBG] from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:39:04.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:04 smithi077 bash[80681]: cluster 2024-09-13T00:39:03.340674+0000 mon.a (mon.0) 38 : cluster [DBG] mgrmap e37: y(active, since 5s), standbys: x 2024-09-13T00:39:04.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:04 smithi077 bash[80681]: audit 2024-09-13T00:39:03.795270+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:04.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:04 smithi077 bash[80681]: audit 2024-09-13T00:39:03.801711+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:04.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:04 smithi077 bash[80681]: cluster 2024-09-13T00:39:03.836356+0000 mon.a (mon.0) 41 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2024-09-13T00:39:04.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:04 smithi077 bash[32040]: audit 2024-09-13T00:39:03.266194+0000 mgr.y (mgr.25304) 9 : audit [DBG] from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:39:04.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:04 smithi077 bash[32040]: cluster 2024-09-13T00:39:03.340674+0000 mon.a (mon.0) 38 : cluster [DBG] mgrmap e37: y(active, since 5s), standbys: x 2024-09-13T00:39:04.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:04 smithi077 bash[32040]: audit 2024-09-13T00:39:03.795270+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:04.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:04 smithi077 bash[32040]: audit 2024-09-13T00:39:03.801711+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:04.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:04 smithi077 bash[32040]: cluster 2024-09-13T00:39:03.836356+0000 mon.a (mon.0) 41 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2024-09-13T00:39:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:04 smithi088 bash[24895]: audit 2024-09-13T00:39:03.266194+0000 mgr.y (mgr.25304) 9 : audit [DBG] from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:39:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:04 smithi088 bash[24895]: cluster 2024-09-13T00:39:03.340674+0000 mon.a (mon.0) 38 : cluster [DBG] mgrmap e37: y(active, since 5s), standbys: x 2024-09-13T00:39:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:04 smithi088 bash[24895]: audit 2024-09-13T00:39:03.795270+0000 mon.a (mon.0) 39 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:04 smithi088 bash[24895]: audit 2024-09-13T00:39:03.801711+0000 mon.a (mon.0) 40 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:04 smithi088 bash[24895]: cluster 2024-09-13T00:39:03.836356+0000 mon.a (mon.0) 41 : cluster [DBG] osdmap e296: 8 total, 8 up, 8 in 2024-09-13T00:39:04.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:04 smithi088 bash[61791]: ts=2024-09-13T00:39:04.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:04.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:04 smithi088 bash[61791]: ts=2024-09-13T00:39:04.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:05.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:05 smithi088 bash[24895]: cluster 2024-09-13T00:39:04.320622+0000 mgr.y (mgr.25304) 10 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 7.4 KiB/s rd, 2.0 KiB/s wr, 19 op/s 2024-09-13T00:39:05.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:05 smithi088 bash[24895]: audit 2024-09-13T00:39:04.491649+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:05.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:05 smithi088 bash[24895]: audit 2024-09-13T00:39:04.501696+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:05.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:05 smithi088 bash[24895]: audit 2024-09-13T00:39:04.709527+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:05.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:05 smithi088 bash[24895]: audit 2024-09-13T00:39:04.724609+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:05.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:05 smithi088 bash[24895]: cluster 2024-09-13T00:39:04.856093+0000 mon.a (mon.0) 46 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2024-09-13T00:39:05.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:05 smithi077 bash[80681]: cluster 2024-09-13T00:39:04.320622+0000 mgr.y (mgr.25304) 10 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 7.4 KiB/s rd, 2.0 KiB/s wr, 19 op/s 2024-09-13T00:39:05.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:05 smithi077 bash[80681]: audit 2024-09-13T00:39:04.491649+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:05.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:05 smithi077 bash[80681]: audit 2024-09-13T00:39:04.501696+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:05.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:05 smithi077 bash[80681]: audit 2024-09-13T00:39:04.709527+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:05.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:05 smithi077 bash[80681]: audit 2024-09-13T00:39:04.724609+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:05.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:05 smithi077 bash[80681]: cluster 2024-09-13T00:39:04.856093+0000 mon.a (mon.0) 46 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2024-09-13T00:39:05.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:05 smithi077 bash[32040]: cluster 2024-09-13T00:39:04.320622+0000 mgr.y (mgr.25304) 10 : cluster [DBG] pgmap v12: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 7.4 KiB/s rd, 2.0 KiB/s wr, 19 op/s 2024-09-13T00:39:05.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:05 smithi077 bash[32040]: audit 2024-09-13T00:39:04.491649+0000 mon.a (mon.0) 42 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:05.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:05 smithi077 bash[32040]: audit 2024-09-13T00:39:04.501696+0000 mon.a (mon.0) 43 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:05.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:05 smithi077 bash[32040]: audit 2024-09-13T00:39:04.709527+0000 mon.a (mon.0) 44 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:05.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:05 smithi077 bash[32040]: audit 2024-09-13T00:39:04.724609+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:05.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:05 smithi077 bash[32040]: cluster 2024-09-13T00:39:04.856093+0000 mon.a (mon.0) 46 : cluster [DBG] osdmap e297: 8 total, 8 up, 8 in 2024-09-13T00:39:05.884 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap1 2024-09-13T00:39:06.027 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:39:06.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:06 smithi077 bash[32040]: audit 2024-09-13T00:39:05.476695+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:06.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:06 smithi077 bash[32040]: audit 2024-09-13T00:39:05.490021+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:06.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:06 smithi077 bash[32040]: cluster 2024-09-13T00:39:05.854408+0000 mon.a (mon.0) 49 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2024-09-13T00:39:06.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:06 smithi077 bash[32040]: cluster 2024-09-13T00:39:06.321355+0000 mgr.y (mgr.25304) 11 : cluster [DBG] pgmap v15: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 463 KiB/s rd, 33 KiB/s wr, 563 op/s 2024-09-13T00:39:06.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:06 smithi077 bash[80681]: audit 2024-09-13T00:39:05.476695+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:06.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:06 smithi077 bash[80681]: audit 2024-09-13T00:39:05.490021+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:06.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:06 smithi077 bash[80681]: cluster 2024-09-13T00:39:05.854408+0000 mon.a (mon.0) 49 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2024-09-13T00:39:06.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:06 smithi077 bash[80681]: cluster 2024-09-13T00:39:06.321355+0000 mgr.y (mgr.25304) 11 : cluster [DBG] pgmap v15: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 463 KiB/s rd, 33 KiB/s wr, 563 op/s 2024-09-13T00:39:06.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:06 smithi088 bash[24895]: audit 2024-09-13T00:39:05.476695+0000 mon.a (mon.0) 47 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:06.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:06 smithi088 bash[24895]: audit 2024-09-13T00:39:05.490021+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:06.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:06 smithi088 bash[24895]: cluster 2024-09-13T00:39:05.854408+0000 mon.a (mon.0) 49 : cluster [DBG] osdmap e298: 8 total, 8 up, 8 in 2024-09-13T00:39:06.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:06 smithi088 bash[24895]: cluster 2024-09-13T00:39:06.321355+0000 mgr.y (mgr.25304) 11 : cluster [DBG] pgmap v15: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 463 KiB/s rd, 33 KiB/s wr, 563 op/s 2024-09-13T00:39:07.242 INFO:tasks.workunit.client.0.smithi077.stdout:migration_abort 2024-09-13T00:39:08.134 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.CloneV2ParentAbortRelinkNotNeeded (4186 ms) 2024-09-13T00:39:08.134 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.StressNoMigrate 2024-09-13T00:39:08.219 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:39:08.249 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 79010491~2538262 2024-09-13T00:39:08.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:07 smithi077 bash[80681]: cluster 2024-09-13T00:39:06.860902+0000 mon.a (mon.0) 50 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2024-09-13T00:39:08.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:07 smithi077 bash[32040]: cluster 2024-09-13T00:39:06.860902+0000 mon.a (mon.0) 50 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2024-09-13T00:39:08.299 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 77268537~6617543 2024-09-13T00:39:08.318 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 48184641~6026265 2024-09-13T00:39:08.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:07 smithi088 bash[24895]: cluster 2024-09-13T00:39:06.860902+0000 mon.a (mon.0) 50 : cluster [DBG] osdmap e299: 8 total, 8 up, 8 in 2024-09-13T00:39:08.389 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 2621081~1322059 2024-09-13T00:39:08.389 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 9619533~2192465 2024-09-13T00:39:08.416 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 69105515~1140762 2024-09-13T00:39:08.416 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 31253354~7582144 2024-09-13T00:39:08.469 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 76770945~3689727 2024-09-13T00:39:08.474 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap0 2024-09-13T00:39:08.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:39:08] "GET /metrics HTTP/1.1" 200 36041 "" "Prometheus/2.43.0" 2024-09-13T00:39:08.888 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:39:09.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:08 smithi088 bash[24895]: cluster 2024-09-13T00:39:07.873756+0000 mon.a (mon.0) 51 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2024-09-13T00:39:09.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:08 smithi088 bash[24895]: cluster 2024-09-13T00:39:08.322306+0000 mgr.y (mgr.25304) 12 : cluster [DBG] pgmap v18: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 456 KiB/s rd, 31 KiB/s wr, 543 op/s 2024-09-13T00:39:09.101 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:08 smithi088 bash[61791]: ts=2024-09-13T00:39:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:09.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:08 smithi077 bash[80681]: cluster 2024-09-13T00:39:07.873756+0000 mon.a (mon.0) 51 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2024-09-13T00:39:09.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:08 smithi077 bash[80681]: cluster 2024-09-13T00:39:08.322306+0000 mgr.y (mgr.25304) 12 : cluster [DBG] pgmap v18: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 456 KiB/s rd, 31 KiB/s wr, 543 op/s 2024-09-13T00:39:09.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:08 smithi077 bash[32040]: cluster 2024-09-13T00:39:07.873756+0000 mon.a (mon.0) 51 : cluster [DBG] osdmap e300: 8 total, 8 up, 8 in 2024-09-13T00:39:09.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:08 smithi077 bash[32040]: cluster 2024-09-13T00:39:08.322306+0000 mgr.y (mgr.25304) 12 : cluster [DBG] pgmap v18: 137 pgs: 137 active+clean; 901 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 456 KiB/s rd, 31 KiB/s wr, 543 op/s 2024-09-13T00:39:09.916 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap0 2024-09-13T00:39:09.948 INFO:tasks.workunit.client.0.smithi077.stdout:clone image157 -> image160 2024-09-13T00:39:10.181 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 94440280 2024-09-13T00:39:10.211 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 87293307~7146973 2024-09-13T00:39:10.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:09 smithi077 bash[80681]: cluster 2024-09-13T00:39:08.883749+0000 mon.a (mon.0) 52 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2024-09-13T00:39:10.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:09 smithi077 bash[32040]: cluster 2024-09-13T00:39:08.883749+0000 mon.a (mon.0) 52 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2024-09-13T00:39:10.295 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 60489590~5079641 2024-09-13T00:39:10.308 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 63137594~7768077 2024-09-13T00:39:10.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:09 smithi088 bash[24895]: cluster 2024-09-13T00:39:08.883749+0000 mon.a (mon.0) 52 : cluster [DBG] osdmap e301: 8 total, 8 up, 8 in 2024-09-13T00:39:10.370 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 8275089~3537258 2024-09-13T00:39:10.390 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 89029349~1312737 2024-09-13T00:39:10.401 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 62558648~2722067 2024-09-13T00:39:10.404 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 61420950~7955291 2024-09-13T00:39:10.449 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 26804462~4506680 2024-09-13T00:39:10.477 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:39:11.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:10 smithi077 bash[32040]: cluster 2024-09-13T00:39:09.886627+0000 mon.a (mon.0) 53 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2024-09-13T00:39:11.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:10 smithi077 bash[32040]: cluster 2024-09-13T00:39:10.323005+0000 mgr.y (mgr.25304) 13 : cluster [DBG] pgmap v21: 137 pgs: 137 active+clean; 943 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 269 KiB/s rd, 8.8 MiB/s wr, 362 op/s 2024-09-13T00:39:11.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:10 smithi077 bash[32040]: cluster 2024-09-13T00:39:10.828599+0000 mon.a (mon.0) 54 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2024-09-13T00:39:11.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:10 smithi077 bash[80681]: cluster 2024-09-13T00:39:09.886627+0000 mon.a (mon.0) 53 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2024-09-13T00:39:11.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:10 smithi077 bash[80681]: cluster 2024-09-13T00:39:10.323005+0000 mgr.y (mgr.25304) 13 : cluster [DBG] pgmap v21: 137 pgs: 137 active+clean; 943 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 269 KiB/s rd, 8.8 MiB/s wr, 362 op/s 2024-09-13T00:39:11.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:10 smithi077 bash[80681]: cluster 2024-09-13T00:39:10.828599+0000 mon.a (mon.0) 54 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2024-09-13T00:39:11.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:10 smithi088 bash[24895]: cluster 2024-09-13T00:39:09.886627+0000 mon.a (mon.0) 53 : cluster [DBG] osdmap e302: 8 total, 8 up, 8 in 2024-09-13T00:39:11.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:10 smithi088 bash[24895]: cluster 2024-09-13T00:39:10.323005+0000 mgr.y (mgr.25304) 13 : cluster [DBG] pgmap v21: 137 pgs: 137 active+clean; 943 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 269 KiB/s rd, 8.8 MiB/s wr, 362 op/s 2024-09-13T00:39:11.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:10 smithi088 bash[24895]: cluster 2024-09-13T00:39:10.828599+0000 mon.a (mon.0) 54 : cluster [DBG] osdmap e303: 8 total, 8 up, 8 in 2024-09-13T00:39:11.854 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 93891073 2024-09-13T00:39:11.896 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 69041598~6493825 2024-09-13T00:39:11.926 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 10171021~5012902 2024-09-13T00:39:11.926 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 24523108~6548789 2024-09-13T00:39:11.946 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 88557563~5333510 2024-09-13T00:39:11.946 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 88411161~5479912 2024-09-13T00:39:11.994 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 21803317~5709371 2024-09-13T00:39:11.995 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 85058745~3274196 2024-09-13T00:39:11.997 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 62209873~4336538 2024-09-13T00:39:11.998 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap2 2024-09-13T00:39:12.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:12 smithi088 bash[61791]: ts=2024-09-13T00:39:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:12.790 INFO:teuthology.orchestra.run.smithi077.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:39:13.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:12 smithi088 bash[24895]: cluster 2024-09-13T00:39:11.832134+0000 mon.a (mon.0) 55 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2024-09-13T00:39:13.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:12 smithi088 bash[24895]: cluster 2024-09-13T00:39:12.323744+0000 mgr.y (mgr.25304) 14 : cluster [DBG] pgmap v24: 137 pgs: 137 active+clean; 943 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 269 KiB/s rd, 8.8 MiB/s wr, 362 op/s 2024-09-13T00:39:13.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:12 smithi088 bash[24895]: audit 2024-09-13T00:39:12.790980+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:13.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:12 smithi077 bash[80681]: cluster 2024-09-13T00:39:11.832134+0000 mon.a (mon.0) 55 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2024-09-13T00:39:13.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:12 smithi077 bash[80681]: cluster 2024-09-13T00:39:12.323744+0000 mgr.y (mgr.25304) 14 : cluster [DBG] pgmap v24: 137 pgs: 137 active+clean; 943 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 269 KiB/s rd, 8.8 MiB/s wr, 362 op/s 2024-09-13T00:39:13.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:12 smithi077 bash[80681]: audit 2024-09-13T00:39:12.790980+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:13.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:12 smithi077 bash[32040]: cluster 2024-09-13T00:39:11.832134+0000 mon.a (mon.0) 55 : cluster [DBG] osdmap e304: 8 total, 8 up, 8 in 2024-09-13T00:39:13.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:12 smithi077 bash[32040]: cluster 2024-09-13T00:39:12.323744+0000 mgr.y (mgr.25304) 14 : cluster [DBG] pgmap v24: 137 pgs: 137 active+clean; 943 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 269 KiB/s rd, 8.8 MiB/s wr, 362 op/s 2024-09-13T00:39:13.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:12 smithi077 bash[32040]: audit 2024-09-13T00:39:12.790980+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:13.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:13 smithi088 bash[61791]: ts=2024-09-13T00:39:13.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:13.678 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'sleep 60' 2024-09-13T00:39:13.870 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 6349887~4797124 2024-09-13T00:39:13.887 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 33067256~8174782 2024-09-13T00:39:13.887 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 28610338~3916116 2024-09-13T00:39:13.893 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 87571743~1392229 2024-09-13T00:39:13.893 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 91709083~2181990 2024-09-13T00:39:13.894 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 55793360~5635339 2024-09-13T00:39:13.915 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 3740556~7634125 2024-09-13T00:39:13.939 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 83908710~705193 2024-09-13T00:39:13.939 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap3 2024-09-13T00:39:14.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:13 smithi077 bash[80681]: audit 2024-09-13T00:39:12.783215+0000 mgr.y (mgr.25304) 15 : audit [DBG] from='client.25355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:39:14.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:13 smithi077 bash[80681]: cephadm 2024-09-13T00:39:12.783667+0000 mgr.y (mgr.25304) 16 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:39:14.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:13 smithi077 bash[80681]: cluster 2024-09-13T00:39:12.839283+0000 mon.a (mon.0) 57 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2024-09-13T00:39:14.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:13 smithi077 bash[80681]: audit 2024-09-13T00:39:13.404789+0000 mon.b (mon.1) 117 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:14.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:13 smithi077 bash[32040]: audit 2024-09-13T00:39:12.783215+0000 mgr.y (mgr.25304) 15 : audit [DBG] from='client.25355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:39:14.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:13 smithi077 bash[32040]: cephadm 2024-09-13T00:39:12.783667+0000 mgr.y (mgr.25304) 16 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:39:14.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:13 smithi077 bash[32040]: cluster 2024-09-13T00:39:12.839283+0000 mon.a (mon.0) 57 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2024-09-13T00:39:14.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:13 smithi077 bash[32040]: audit 2024-09-13T00:39:13.404789+0000 mon.b (mon.1) 117 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:14.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:13 smithi088 bash[24895]: audit 2024-09-13T00:39:12.783215+0000 mgr.y (mgr.25304) 15 : audit [DBG] from='client.25355 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:39:14.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:13 smithi088 bash[24895]: cephadm 2024-09-13T00:39:12.783667+0000 mgr.y (mgr.25304) 16 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:39:14.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:13 smithi088 bash[24895]: cluster 2024-09-13T00:39:12.839283+0000 mon.a (mon.0) 57 : cluster [DBG] osdmap e305: 8 total, 8 up, 8 in 2024-09-13T00:39:14.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:13 smithi088 bash[24895]: audit 2024-09-13T00:39:13.404789+0000 mon.b (mon.1) 117 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:14.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:14 smithi088 bash[61791]: ts=2024-09-13T00:39:14.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:14.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:14 smithi088 bash[61791]: ts=2024-09-13T00:39:14.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:15.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:14 smithi077 bash[32040]: cluster 2024-09-13T00:39:13.849924+0000 mon.a (mon.0) 58 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2024-09-13T00:39:15.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:14 smithi077 bash[32040]: cluster 2024-09-13T00:39:14.324964+0000 mgr.y (mgr.25304) 17 : cluster [DBG] pgmap v27: 137 pgs: 137 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 19 MiB/s wr, 161 op/s 2024-09-13T00:39:15.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:14 smithi077 bash[80681]: cluster 2024-09-13T00:39:13.849924+0000 mon.a (mon.0) 58 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2024-09-13T00:39:15.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:14 smithi077 bash[80681]: cluster 2024-09-13T00:39:14.324964+0000 mgr.y (mgr.25304) 17 : cluster [DBG] pgmap v27: 137 pgs: 137 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 19 MiB/s wr, 161 op/s 2024-09-13T00:39:15.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:14 smithi088 bash[24895]: cluster 2024-09-13T00:39:13.849924+0000 mon.a (mon.0) 58 : cluster [DBG] osdmap e306: 8 total, 8 up, 8 in 2024-09-13T00:39:15.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:14 smithi088 bash[24895]: cluster 2024-09-13T00:39:14.324964+0000 mgr.y (mgr.25304) 17 : cluster [DBG] pgmap v27: 137 pgs: 137 active+clean; 1.0 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 2.1 MiB/s rd, 19 MiB/s wr, 161 op/s 2024-09-13T00:39:16.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: cluster 2024-09-13T00:39:14.851939+0000 mon.a (mon.0) 59 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2024-09-13T00:39:16.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: audit 2024-09-13T00:39:14.958825+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: audit 2024-09-13T00:39:14.972295+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: audit 2024-09-13T00:39:14.976086+0000 mon.b (mon.1) 118 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:39:16.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: audit 2024-09-13T00:39:14.978396+0000 mon.b (mon.1) 119 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:39:16.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: cephadm 2024-09-13T00:39:14.982064+0000 mgr.y (mgr.25304) 18 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.conf 2024-09-13T00:39:16.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: cephadm 2024-09-13T00:39:14.982661+0000 mgr.y (mgr.25304) 19 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:39:16.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: cephadm 2024-09-13T00:39:15.178101+0000 mgr.y (mgr.25304) 20 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:39:16.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: cephadm 2024-09-13T00:39:15.183847+0000 mgr.y (mgr.25304) 21 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:39:16.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: cephadm 2024-09-13T00:39:15.305929+0000 mgr.y (mgr.25304) 22 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:39:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: cephadm 2024-09-13T00:39:15.326208+0000 mgr.y (mgr.25304) 23 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:39:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: audit 2024-09-13T00:39:15.561939+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: audit 2024-09-13T00:39:15.571944+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: audit 2024-09-13T00:39:15.610370+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: audit 2024-09-13T00:39:15.618477+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: audit 2024-09-13T00:39:15.627920+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: audit 2024-09-13T00:39:15.755415+0000 mon.b (mon.1) 120 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:39:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: audit 2024-09-13T00:39:15.758573+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:39:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: audit 2024-09-13T00:39:15.760432+0000 mon.b (mon.1) 122 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:39:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: audit 2024-09-13T00:39:15.771829+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[32040]: cluster 2024-09-13T00:39:15.834394+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2024-09-13T00:39:16.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: cluster 2024-09-13T00:39:14.851939+0000 mon.a (mon.0) 59 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2024-09-13T00:39:16.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: audit 2024-09-13T00:39:14.958825+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: audit 2024-09-13T00:39:14.972295+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: audit 2024-09-13T00:39:14.976086+0000 mon.b (mon.1) 118 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:39:16.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: audit 2024-09-13T00:39:14.978396+0000 mon.b (mon.1) 119 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:39:16.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: cephadm 2024-09-13T00:39:14.982064+0000 mgr.y (mgr.25304) 18 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.conf 2024-09-13T00:39:16.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: cephadm 2024-09-13T00:39:14.982661+0000 mgr.y (mgr.25304) 19 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:39:16.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: cephadm 2024-09-13T00:39:15.178101+0000 mgr.y (mgr.25304) 20 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:39:16.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: cephadm 2024-09-13T00:39:15.183847+0000 mgr.y (mgr.25304) 21 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:39:16.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: cephadm 2024-09-13T00:39:15.305929+0000 mgr.y (mgr.25304) 22 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:39:16.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: cephadm 2024-09-13T00:39:15.326208+0000 mgr.y (mgr.25304) 23 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:39:16.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: audit 2024-09-13T00:39:15.561939+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: audit 2024-09-13T00:39:15.571944+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: audit 2024-09-13T00:39:15.610370+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: audit 2024-09-13T00:39:15.618477+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: audit 2024-09-13T00:39:15.627920+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: audit 2024-09-13T00:39:15.755415+0000 mon.b (mon.1) 120 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:39:16.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: audit 2024-09-13T00:39:15.758573+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:39:16.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: audit 2024-09-13T00:39:15.760432+0000 mon.b (mon.1) 122 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:39:16.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: audit 2024-09-13T00:39:15.771829+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:15 smithi077 bash[80681]: cluster 2024-09-13T00:39:15.834394+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2024-09-13T00:39:16.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: cluster 2024-09-13T00:39:14.851939+0000 mon.a (mon.0) 59 : cluster [DBG] osdmap e307: 8 total, 8 up, 8 in 2024-09-13T00:39:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: audit 2024-09-13T00:39:14.958825+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: audit 2024-09-13T00:39:14.972295+0000 mon.a (mon.0) 61 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: audit 2024-09-13T00:39:14.976086+0000 mon.b (mon.1) 118 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:39:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: audit 2024-09-13T00:39:14.978396+0000 mon.b (mon.1) 119 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:39:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: cephadm 2024-09-13T00:39:14.982064+0000 mgr.y (mgr.25304) 18 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.conf 2024-09-13T00:39:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: cephadm 2024-09-13T00:39:14.982661+0000 mgr.y (mgr.25304) 19 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.conf 2024-09-13T00:39:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: cephadm 2024-09-13T00:39:15.178101+0000 mgr.y (mgr.25304) 20 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:39:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: cephadm 2024-09-13T00:39:15.183847+0000 mgr.y (mgr.25304) 21 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-13T00:39:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: cephadm 2024-09-13T00:39:15.305929+0000 mgr.y (mgr.25304) 22 : cephadm [INF] Updating smithi077:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:39:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: cephadm 2024-09-13T00:39:15.326208+0000 mgr.y (mgr.25304) 23 : cephadm [INF] Updating smithi088:/etc/ceph/ceph.client.admin.keyring 2024-09-13T00:39:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: audit 2024-09-13T00:39:15.561939+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: audit 2024-09-13T00:39:15.571944+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: audit 2024-09-13T00:39:15.610370+0000 mon.a (mon.0) 64 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: audit 2024-09-13T00:39:15.618477+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: audit 2024-09-13T00:39:15.627920+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: audit 2024-09-13T00:39:15.755415+0000 mon.b (mon.1) 120 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:39:16.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: audit 2024-09-13T00:39:15.758573+0000 mon.b (mon.1) 121 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:39:16.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: audit 2024-09-13T00:39:15.760432+0000 mon.b (mon.1) 122 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:39:16.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: audit 2024-09-13T00:39:15.771829+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:39:16.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:15 smithi088 bash[24895]: cluster 2024-09-13T00:39:15.834394+0000 mon.a (mon.0) 68 : cluster [DBG] osdmap e308: 8 total, 8 up, 8 in 2024-09-13T00:39:16.695 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.StressNoMigrate (8562 ms) 2024-09-13T00:39:16.695 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.Stress 2024-09-13T00:39:16.780 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:39:16.809 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 33485870~76250 2024-09-13T00:39:16.835 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 13944771~715282 2024-09-13T00:39:16.835 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 6424653~4447269 2024-09-13T00:39:16.887 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 66441721~4813060 2024-09-13T00:39:16.887 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 59481246~6983952 2024-09-13T00:39:16.957 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 82420192~592724 2024-09-13T00:39:16.957 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 72498712~7623034 2024-09-13T00:39:17.035 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 7409208~8116769 2024-09-13T00:39:17.052 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap0 2024-09-13T00:39:17.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:16 smithi077 bash[80681]: cephadm 2024-09-13T00:39:15.430661+0000 mgr.y (mgr.25304) 24 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:39:17.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:16 smithi077 bash[80681]: cephadm 2024-09-13T00:39:15.467807+0000 mgr.y (mgr.25304) 25 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:39:17.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:16 smithi077 bash[80681]: cluster 2024-09-13T00:39:16.325853+0000 mgr.y (mgr.25304) 26 : cluster [DBG] pgmap v30: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 4.9 MiB/s rd, 34 MiB/s wr, 383 op/s 2024-09-13T00:39:17.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:16 smithi077 bash[32040]: cephadm 2024-09-13T00:39:15.430661+0000 mgr.y (mgr.25304) 24 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:39:17.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:16 smithi077 bash[32040]: cephadm 2024-09-13T00:39:15.467807+0000 mgr.y (mgr.25304) 25 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:39:17.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:16 smithi077 bash[32040]: cluster 2024-09-13T00:39:16.325853+0000 mgr.y (mgr.25304) 26 : cluster [DBG] pgmap v30: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 4.9 MiB/s rd, 34 MiB/s wr, 383 op/s 2024-09-13T00:39:17.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:16 smithi088 bash[24895]: cephadm 2024-09-13T00:39:15.430661+0000 mgr.y (mgr.25304) 24 : cephadm [INF] Updating smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:39:17.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:16 smithi088 bash[24895]: cephadm 2024-09-13T00:39:15.467807+0000 mgr.y (mgr.25304) 25 : cephadm [INF] Updating smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-13T00:39:17.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:16 smithi088 bash[24895]: cluster 2024-09-13T00:39:16.325853+0000 mgr.y (mgr.25304) 26 : cluster [DBG] pgmap v30: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 4.9 MiB/s rd, 34 MiB/s wr, 383 op/s 2024-09-13T00:39:18.474 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:39:18.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:39:18] "GET /metrics HTTP/1.1" 200 38886 "" "Prometheus/2.43.0" 2024-09-13T00:39:18.919 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap0 2024-09-13T00:39:18.950 INFO:tasks.workunit.client.0.smithi077.stdout:clone image161 -> image164 2024-09-13T00:39:19.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:18 smithi088 bash[24895]: cluster 2024-09-13T00:39:17.882709+0000 mon.a (mon.0) 69 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2024-09-13T00:39:19.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:18 smithi088 bash[24895]: cluster 2024-09-13T00:39:18.326658+0000 mgr.y (mgr.25304) 27 : cluster [DBG] pgmap v32: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 4.4 MiB/s rd, 31 MiB/s wr, 342 op/s 2024-09-13T00:39:19.102 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:18 smithi088 bash[61791]: ts=2024-09-13T00:39:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:19.200 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 33773447~5084519 2024-09-13T00:39:19.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:18 smithi077 bash[80681]: cluster 2024-09-13T00:39:17.882709+0000 mon.a (mon.0) 69 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2024-09-13T00:39:19.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:18 smithi077 bash[80681]: cluster 2024-09-13T00:39:18.326658+0000 mgr.y (mgr.25304) 27 : cluster [DBG] pgmap v32: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 4.4 MiB/s rd, 31 MiB/s wr, 342 op/s 2024-09-13T00:39:19.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:18 smithi077 bash[32040]: cluster 2024-09-13T00:39:17.882709+0000 mon.a (mon.0) 69 : cluster [DBG] osdmap e309: 8 total, 8 up, 8 in 2024-09-13T00:39:19.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:18 smithi077 bash[32040]: cluster 2024-09-13T00:39:18.326658+0000 mgr.y (mgr.25304) 27 : cluster [DBG] pgmap v32: 137 pgs: 137 active+clean; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 4.4 MiB/s rd, 31 MiB/s wr, 342 op/s 2024-09-13T00:39:19.280 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 7266509~5406873 2024-09-13T00:39:19.310 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 52985252~4071162 2024-09-13T00:39:19.357 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 11705287~3419565 2024-09-13T00:39:19.363 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 4124758~6394160 2024-09-13T00:39:19.422 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 76310981~7575099 2024-09-13T00:39:19.462 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 24428565~2990133 2024-09-13T00:39:19.499 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 12094023~923526 2024-09-13T00:39:19.505 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:39:20.167 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:19 smithi077 bash[80681]: cluster 2024-09-13T00:39:18.895647+0000 mon.a (mon.0) 70 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2024-09-13T00:39:20.167 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:19 smithi077 bash[32040]: cluster 2024-09-13T00:39:18.895647+0000 mon.a (mon.0) 70 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2024-09-13T00:39:20.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:19 smithi088 bash[24895]: cluster 2024-09-13T00:39:18.895647+0000 mon.a (mon.0) 70 : cluster [DBG] osdmap e310: 8 total, 8 up, 8 in 2024-09-13T00:39:20.933 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 102404757 2024-09-13T00:39:20.967 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 76718203~1243356 2024-09-13T00:39:20.980 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 32075856~336805 2024-09-13T00:39:20.980 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 89404145~5296377 2024-09-13T00:39:20.990 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 85546303~1238773 2024-09-13T00:39:20.990 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 5950429~6987687 2024-09-13T00:39:21.014 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 73301816~1124403 2024-09-13T00:39:21.015 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 75566102~5343892 2024-09-13T00:39:21.031 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 67529039~7650230 2024-09-13T00:39:21.117 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap2 2024-09-13T00:39:21.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:20 smithi077 bash[32040]: cluster 2024-09-13T00:39:19.896305+0000 mon.a (mon.0) 71 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2024-09-13T00:39:21.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:20 smithi077 bash[32040]: cluster 2024-09-13T00:39:20.327263+0000 mgr.y (mgr.25304) 28 : cluster [DBG] pgmap v35: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 59 MiB/s rd, 11 MiB/s wr, 267 op/s 2024-09-13T00:39:21.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:20 smithi077 bash[80681]: cluster 2024-09-13T00:39:19.896305+0000 mon.a (mon.0) 71 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2024-09-13T00:39:21.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:20 smithi077 bash[80681]: cluster 2024-09-13T00:39:20.327263+0000 mgr.y (mgr.25304) 28 : cluster [DBG] pgmap v35: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 59 MiB/s rd, 11 MiB/s wr, 267 op/s 2024-09-13T00:39:21.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:20 smithi088 bash[24895]: cluster 2024-09-13T00:39:19.896305+0000 mon.a (mon.0) 71 : cluster [DBG] osdmap e311: 8 total, 8 up, 8 in 2024-09-13T00:39:21.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:20 smithi088 bash[24895]: cluster 2024-09-13T00:39:20.327263+0000 mgr.y (mgr.25304) 28 : cluster [DBG] pgmap v35: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 59 MiB/s rd, 11 MiB/s wr, 267 op/s 2024-09-13T00:39:22.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:21 smithi077 bash[80681]: cluster 2024-09-13T00:39:20.916394+0000 mon.a (mon.0) 72 : cluster [DBG] osdmap e312: 8 total, 8 up, 8 in 2024-09-13T00:39:22.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:21 smithi077 bash[32040]: cluster 2024-09-13T00:39:20.916394+0000 mon.a (mon.0) 72 : cluster [DBG] osdmap e312: 8 total, 8 up, 8 in 2024-09-13T00:39:22.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:21 smithi088 bash[24895]: cluster 2024-09-13T00:39:20.916394+0000 mon.a (mon.0) 72 : cluster [DBG] osdmap e312: 8 total, 8 up, 8 in 2024-09-13T00:39:22.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:22 smithi088 bash[61791]: ts=2024-09-13T00:39:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:22.958 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap2 2024-09-13T00:39:22.993 INFO:tasks.workunit.client.0.smithi077.stdout:clone image164 -> image166 2024-09-13T00:39:23.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:22 smithi077 bash[80681]: cluster 2024-09-13T00:39:21.916920+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2024-09-13T00:39:23.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:22 smithi077 bash[80681]: cluster 2024-09-13T00:39:22.327727+0000 mgr.y (mgr.25304) 29 : cluster [DBG] pgmap v38: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 67 MiB/s rd, 12 MiB/s wr, 300 op/s 2024-09-13T00:39:23.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:22 smithi077 bash[32040]: cluster 2024-09-13T00:39:21.916920+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2024-09-13T00:39:23.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:22 smithi077 bash[32040]: cluster 2024-09-13T00:39:22.327727+0000 mgr.y (mgr.25304) 29 : cluster [DBG] pgmap v38: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 67 MiB/s rd, 12 MiB/s wr, 300 op/s 2024-09-13T00:39:23.327 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 8412563~7641359 2024-09-13T00:39:23.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:22 smithi088 bash[24895]: cluster 2024-09-13T00:39:21.916920+0000 mon.a (mon.0) 73 : cluster [DBG] osdmap e313: 8 total, 8 up, 8 in 2024-09-13T00:39:23.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:22 smithi088 bash[24895]: cluster 2024-09-13T00:39:22.327727+0000 mgr.y (mgr.25304) 29 : cluster [DBG] pgmap v38: 137 pgs: 137 active+clean; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 67 MiB/s rd, 12 MiB/s wr, 300 op/s 2024-09-13T00:39:23.445 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 6541734~8309387 2024-09-13T00:39:23.501 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 71971326~7917751 2024-09-13T00:39:23.574 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 64159288~6367642 2024-09-13T00:39:23.615 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 12918586~1836394 2024-09-13T00:39:23.638 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 48194138~3610789 2024-09-13T00:39:23.676 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 86042674~3425911 2024-09-13T00:39:23.751 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 43645598~3522234 2024-09-13T00:39:23.765 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap3 2024-09-13T00:39:23.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:23 smithi088 bash[61791]: ts=2024-09-13T00:39:23.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:24.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:23 smithi077 bash[80681]: cluster 2024-09-13T00:39:22.937753+0000 mon.a (mon.0) 74 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2024-09-13T00:39:24.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:23 smithi077 bash[32040]: cluster 2024-09-13T00:39:22.937753+0000 mon.a (mon.0) 74 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2024-09-13T00:39:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:23 smithi088 bash[24895]: cluster 2024-09-13T00:39:22.937753+0000 mon.a (mon.0) 74 : cluster [DBG] osdmap e314: 8 total, 8 up, 8 in 2024-09-13T00:39:24.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:24 smithi088 bash[61791]: ts=2024-09-13T00:39:24.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:24.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:24 smithi088 bash[61791]: ts=2024-09-13T00:39:24.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:24.986 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 114531066 2024-09-13T00:39:25.028 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:39:25.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:24 smithi077 bash[80681]: cluster 2024-09-13T00:39:23.938954+0000 mon.a (mon.0) 75 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2024-09-13T00:39:25.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:24 smithi077 bash[80681]: cluster 2024-09-13T00:39:24.328911+0000 mgr.y (mgr.25304) 30 : cluster [DBG] pgmap v41: 137 pgs: 137 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 6.9 MiB/s rd, 18 MiB/s wr, 155 op/s 2024-09-13T00:39:25.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:24 smithi077 bash[32040]: cluster 2024-09-13T00:39:23.938954+0000 mon.a (mon.0) 75 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2024-09-13T00:39:25.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:24 smithi077 bash[32040]: cluster 2024-09-13T00:39:24.328911+0000 mgr.y (mgr.25304) 30 : cluster [DBG] pgmap v41: 137 pgs: 137 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 6.9 MiB/s rd, 18 MiB/s wr, 155 op/s 2024-09-13T00:39:25.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:24 smithi088 bash[24895]: cluster 2024-09-13T00:39:23.938954+0000 mon.a (mon.0) 75 : cluster [DBG] osdmap e315: 8 total, 8 up, 8 in 2024-09-13T00:39:25.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:24 smithi088 bash[24895]: cluster 2024-09-13T00:39:24.328911+0000 mgr.y (mgr.25304) 30 : cluster [DBG] pgmap v41: 137 pgs: 137 active+clean; 1.2 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 6.9 MiB/s rd, 18 MiB/s wr, 155 op/s 2024-09-13T00:39:26.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:25 smithi077 bash[80681]: cluster 2024-09-13T00:39:24.957830+0000 mon.a (mon.0) 76 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2024-09-13T00:39:26.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:25 smithi077 bash[32040]: cluster 2024-09-13T00:39:24.957830+0000 mon.a (mon.0) 76 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2024-09-13T00:39:26.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:25 smithi088 bash[24895]: cluster 2024-09-13T00:39:24.957830+0000 mon.a (mon.0) 76 : cluster [DBG] osdmap e316: 8 total, 8 up, 8 in 2024-09-13T00:39:26.999 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:39:27.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:26 smithi077 bash[80681]: cluster 2024-09-13T00:39:25.968098+0000 mon.a (mon.0) 77 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2024-09-13T00:39:27.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:26 smithi077 bash[80681]: cluster 2024-09-13T00:39:26.329601+0000 mgr.y (mgr.25304) 31 : cluster [DBG] pgmap v44: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 12 MiB/s rd, 32 MiB/s wr, 375 op/s 2024-09-13T00:39:27.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:26 smithi077 bash[32040]: cluster 2024-09-13T00:39:25.968098+0000 mon.a (mon.0) 77 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2024-09-13T00:39:27.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:26 smithi077 bash[32040]: cluster 2024-09-13T00:39:26.329601+0000 mgr.y (mgr.25304) 31 : cluster [DBG] pgmap v44: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 12 MiB/s rd, 32 MiB/s wr, 375 op/s 2024-09-13T00:39:27.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:26 smithi088 bash[24895]: cluster 2024-09-13T00:39:25.968098+0000 mon.a (mon.0) 77 : cluster [DBG] osdmap e317: 8 total, 8 up, 8 in 2024-09-13T00:39:27.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:26 smithi088 bash[24895]: cluster 2024-09-13T00:39:26.329601+0000 mgr.y (mgr.25304) 31 : cluster [DBG] pgmap v44: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 12 MiB/s rd, 32 MiB/s wr, 375 op/s 2024-09-13T00:39:27.504 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:39:28.615 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.Stress (11920 ms) 2024-09-13T00:39:28.616 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.Stress2 2024-09-13T00:39:28.699 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:39:28.727 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 19504837~8097265 2024-09-13T00:39:28.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:39:28] "GET /metrics HTTP/1.1" 200 38886 "" "Prometheus/2.43.0" 2024-09-13T00:39:28.805 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 75465503~1232714 2024-09-13T00:39:28.806 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 62196327~3852024 2024-09-13T00:39:28.852 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 21972659~5311923 2024-09-13T00:39:28.859 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 76642581~2737694 2024-09-13T00:39:28.894 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 29818744~2221416 2024-09-13T00:39:28.894 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 44507108~397860 2024-09-13T00:39:28.909 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 79272842~4573699 2024-09-13T00:39:28.915 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap0 2024-09-13T00:39:28.982 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:28 smithi088 bash[61791]: ts=2024-09-13T00:39:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:29.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:28 smithi077 bash[80681]: cluster 2024-09-13T00:39:27.984006+0000 mon.a (mon.0) 78 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2024-09-13T00:39:29.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:28 smithi077 bash[80681]: cluster 2024-09-13T00:39:28.330308+0000 mgr.y (mgr.25304) 32 : cluster [DBG] pgmap v46: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 11 MiB/s rd, 29 MiB/s wr, 341 op/s 2024-09-13T00:39:29.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:28 smithi077 bash[80681]: audit 2024-09-13T00:39:28.405452+0000 mon.b (mon.1) 123 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:29.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:28 smithi077 bash[32040]: cluster 2024-09-13T00:39:27.984006+0000 mon.a (mon.0) 78 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2024-09-13T00:39:29.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:28 smithi077 bash[32040]: cluster 2024-09-13T00:39:28.330308+0000 mgr.y (mgr.25304) 32 : cluster [DBG] pgmap v46: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 11 MiB/s rd, 29 MiB/s wr, 341 op/s 2024-09-13T00:39:29.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:28 smithi077 bash[32040]: audit 2024-09-13T00:39:28.405452+0000 mon.b (mon.1) 123 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:28 smithi088 bash[24895]: cluster 2024-09-13T00:39:27.984006+0000 mon.a (mon.0) 78 : cluster [DBG] osdmap e318: 8 total, 8 up, 8 in 2024-09-13T00:39:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:28 smithi088 bash[24895]: cluster 2024-09-13T00:39:28.330308+0000 mgr.y (mgr.25304) 32 : cluster [DBG] pgmap v46: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 11 MiB/s rd, 29 MiB/s wr, 341 op/s 2024-09-13T00:39:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:28 smithi088 bash[24895]: audit 2024-09-13T00:39:28.405452+0000 mon.b (mon.1) 123 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:30.027 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 144360740 2024-09-13T00:39:30.053 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 34847357~885097 2024-09-13T00:39:30.066 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 97758154~3438854 2024-09-13T00:39:30.067 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 128801665~6141542 2024-09-13T00:39:30.077 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 56103205~961424 2024-09-13T00:39:30.078 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 66156761~2238874 2024-09-13T00:39:30.080 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 66718160~1705172 2024-09-13T00:39:30.119 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 109209065~7528574 2024-09-13T00:39:30.136 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 18627197~6105222 2024-09-13T00:39:30.136 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:39:30.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:29 smithi077 bash[80681]: cluster 2024-09-13T00:39:28.995184+0000 mon.a (mon.0) 79 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2024-09-13T00:39:30.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:29 smithi077 bash[32040]: cluster 2024-09-13T00:39:28.995184+0000 mon.a (mon.0) 79 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2024-09-13T00:39:30.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:29 smithi088 bash[24895]: cluster 2024-09-13T00:39:28.995184+0000 mon.a (mon.0) 79 : cluster [DBG] osdmap e319: 8 total, 8 up, 8 in 2024-09-13T00:39:31.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:31 smithi077 bash[80681]: cluster 2024-09-13T00:39:29.999776+0000 mon.a (mon.0) 80 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2024-09-13T00:39:31.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:31 smithi077 bash[80681]: cluster 2024-09-13T00:39:30.331078+0000 mgr.y (mgr.25304) 33 : cluster [DBG] pgmap v49: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 75 MiB/s rd, 12 MiB/s wr, 775 op/s 2024-09-13T00:39:31.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:31 smithi077 bash[32040]: cluster 2024-09-13T00:39:29.999776+0000 mon.a (mon.0) 80 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2024-09-13T00:39:31.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:31 smithi077 bash[32040]: cluster 2024-09-13T00:39:30.331078+0000 mgr.y (mgr.25304) 33 : cluster [DBG] pgmap v49: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 75 MiB/s rd, 12 MiB/s wr, 775 op/s 2024-09-13T00:39:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:31 smithi088 bash[24895]: cluster 2024-09-13T00:39:29.999776+0000 mon.a (mon.0) 80 : cluster [DBG] osdmap e320: 8 total, 8 up, 8 in 2024-09-13T00:39:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:31 smithi088 bash[24895]: cluster 2024-09-13T00:39:30.331078+0000 mgr.y (mgr.25304) 33 : cluster [DBG] pgmap v49: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 75 MiB/s rd, 12 MiB/s wr, 775 op/s 2024-09-13T00:39:32.037 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap1 2024-09-13T00:39:32.067 INFO:tasks.workunit.client.0.smithi077.stdout:clone image167 -> image170 2024-09-13T00:39:32.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:32 smithi077 bash[80681]: cluster 2024-09-13T00:39:31.010398+0000 mon.a (mon.0) 81 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2024-09-13T00:39:32.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:32 smithi077 bash[32040]: cluster 2024-09-13T00:39:31.010398+0000 mon.a (mon.0) 81 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2024-09-13T00:39:32.310 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 139940819~4419921 2024-09-13T00:39:32.311 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:32 smithi088 bash[24895]: cluster 2024-09-13T00:39:31.010398+0000 mon.a (mon.0) 81 : cluster [DBG] osdmap e321: 8 total, 8 up, 8 in 2024-09-13T00:39:32.385 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 11815822~3930380 2024-09-13T00:39:32.400 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 35120116~7313069 2024-09-13T00:39:32.489 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 118545629~5962891 2024-09-13T00:39:32.504 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 118790165~4768392 2024-09-13T00:39:32.548 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 18643567~3581741 2024-09-13T00:39:32.565 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 574618~3104057 2024-09-13T00:39:32.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:32 smithi088 bash[61791]: ts=2024-09-13T00:39:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:32.608 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 91033160~800704 2024-09-13T00:39:32.620 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap2 2024-09-13T00:39:33.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:33 smithi077 bash[80681]: cluster 2024-09-13T00:39:32.018946+0000 mon.a (mon.0) 82 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2024-09-13T00:39:33.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:33 smithi077 bash[80681]: cluster 2024-09-13T00:39:32.331898+0000 mgr.y (mgr.25304) 34 : cluster [DBG] pgmap v52: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 82 MiB/s rd, 13 MiB/s wr, 845 op/s 2024-09-13T00:39:33.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:33 smithi077 bash[32040]: cluster 2024-09-13T00:39:32.018946+0000 mon.a (mon.0) 82 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2024-09-13T00:39:33.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:33 smithi077 bash[32040]: cluster 2024-09-13T00:39:32.331898+0000 mgr.y (mgr.25304) 34 : cluster [DBG] pgmap v52: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 82 MiB/s rd, 13 MiB/s wr, 845 op/s 2024-09-13T00:39:33.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:33 smithi088 bash[24895]: cluster 2024-09-13T00:39:32.018946+0000 mon.a (mon.0) 82 : cluster [DBG] osdmap e322: 8 total, 8 up, 8 in 2024-09-13T00:39:33.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:33 smithi088 bash[24895]: cluster 2024-09-13T00:39:32.331898+0000 mgr.y (mgr.25304) 34 : cluster [DBG] pgmap v52: 137 pgs: 137 active+clean; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 82 MiB/s rd, 13 MiB/s wr, 845 op/s 2024-09-13T00:39:33.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:33 smithi088 bash[61791]: ts=2024-09-13T00:39:33.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:34.055 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap2 2024-09-13T00:39:34.086 INFO:tasks.workunit.client.0.smithi077.stdout:clone image170 -> image172 2024-09-13T00:39:34.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:34 smithi077 bash[80681]: cluster 2024-09-13T00:39:33.029288+0000 mon.a (mon.0) 83 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2024-09-13T00:39:34.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:34 smithi077 bash[32040]: cluster 2024-09-13T00:39:33.029288+0000 mon.a (mon.0) 83 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2024-09-13T00:39:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:34 smithi088 bash[24895]: cluster 2024-09-13T00:39:33.029288+0000 mon.a (mon.0) 83 : cluster [DBG] osdmap e323: 8 total, 8 up, 8 in 2024-09-13T00:39:34.408 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 79774997~5019069 2024-09-13T00:39:34.497 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 32040355~8217415 2024-09-13T00:39:34.551 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 83282244~966850 2024-09-13T00:39:34.567 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 96877893~3741844 2024-09-13T00:39:34.581 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 20977658~5966773 2024-09-13T00:39:34.664 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 136757318~7603422 2024-09-13T00:39:34.711 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 45194363~2651144 2024-09-13T00:39:34.752 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 71169160~2566028 2024-09-13T00:39:34.769 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap3 2024-09-13T00:39:34.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:34 smithi088 bash[61791]: ts=2024-09-13T00:39:34.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:34.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:34 smithi088 bash[61791]: ts=2024-09-13T00:39:34.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:35.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:35 smithi088 bash[24895]: cluster 2024-09-13T00:39:34.036604+0000 mon.a (mon.0) 84 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2024-09-13T00:39:35.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:35 smithi088 bash[24895]: cluster 2024-09-13T00:39:34.333408+0000 mgr.y (mgr.25304) 35 : cluster [DBG] pgmap v55: 137 pgs: 137 active+clean; 1.4 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 655 KiB/s rd, 15 MiB/s wr, 344 op/s 2024-09-13T00:39:35.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:35 smithi077 bash[80681]: cluster 2024-09-13T00:39:34.036604+0000 mon.a (mon.0) 84 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2024-09-13T00:39:35.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:35 smithi077 bash[80681]: cluster 2024-09-13T00:39:34.333408+0000 mgr.y (mgr.25304) 35 : cluster [DBG] pgmap v55: 137 pgs: 137 active+clean; 1.4 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 655 KiB/s rd, 15 MiB/s wr, 344 op/s 2024-09-13T00:39:35.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:35 smithi077 bash[32040]: cluster 2024-09-13T00:39:34.036604+0000 mon.a (mon.0) 84 : cluster [DBG] osdmap e324: 8 total, 8 up, 8 in 2024-09-13T00:39:35.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:35 smithi077 bash[32040]: cluster 2024-09-13T00:39:34.333408+0000 mgr.y (mgr.25304) 35 : cluster [DBG] pgmap v55: 137 pgs: 137 active+clean; 1.4 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 655 KiB/s rd, 15 MiB/s wr, 344 op/s 2024-09-13T00:39:35.864 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap3 2024-09-13T00:39:35.899 INFO:tasks.workunit.client.0.smithi077.stdout:clone image172 -> image174 2024-09-13T00:39:36.292 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 208102809 2024-09-13T00:39:36.332 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:39:36.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:36 smithi088 bash[24895]: cluster 2024-09-13T00:39:35.047532+0000 mon.a (mon.0) 85 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2024-09-13T00:39:36.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:36 smithi088 bash[24895]: cluster 2024-09-13T00:39:35.841143+0000 mon.a (mon.0) 86 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2024-09-13T00:39:36.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:36 smithi077 bash[80681]: cluster 2024-09-13T00:39:35.047532+0000 mon.a (mon.0) 85 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2024-09-13T00:39:36.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:36 smithi077 bash[80681]: cluster 2024-09-13T00:39:35.841143+0000 mon.a (mon.0) 86 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2024-09-13T00:39:36.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:36 smithi077 bash[32040]: cluster 2024-09-13T00:39:35.047532+0000 mon.a (mon.0) 85 : cluster [DBG] osdmap e325: 8 total, 8 up, 8 in 2024-09-13T00:39:36.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:36 smithi077 bash[32040]: cluster 2024-09-13T00:39:35.841143+0000 mon.a (mon.0) 86 : cluster [DBG] osdmap e326: 8 total, 8 up, 8 in 2024-09-13T00:39:37.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:37 smithi088 bash[24895]: cluster 2024-09-13T00:39:36.334248+0000 mgr.y (mgr.25304) 36 : cluster [DBG] pgmap v58: 137 pgs: 137 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 900 KiB/s rd, 18 MiB/s wr, 393 op/s 2024-09-13T00:39:37.476 INFO:tasks.workunit.client.0.smithi077.stdout:write: a 12980018~6774049 2024-09-13T00:39:37.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:37 smithi077 bash[32040]: cluster 2024-09-13T00:39:36.334248+0000 mgr.y (mgr.25304) 36 : cluster [DBG] pgmap v58: 137 pgs: 137 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 900 KiB/s rd, 18 MiB/s wr, 393 op/s 2024-09-13T00:39:37.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:37 smithi077 bash[80681]: cluster 2024-09-13T00:39:36.334248+0000 mgr.y (mgr.25304) 36 : cluster [DBG] pgmap v58: 137 pgs: 137 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 900 KiB/s rd, 18 MiB/s wr, 393 op/s 2024-09-13T00:39:37.581 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 24247612~3156001 2024-09-13T00:39:37.644 INFO:tasks.workunit.client.0.smithi077.stdout:write: a 1470917~6560760 2024-09-13T00:39:37.738 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 168472107~2839151 2024-09-13T00:39:37.749 INFO:tasks.workunit.client.0.smithi077.stdout:write: a 30669799~5823056 2024-09-13T00:39:37.830 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 82772295~3213108 2024-09-13T00:39:37.890 INFO:tasks.workunit.client.0.smithi077.stdout:write: a 140578768~6012525 2024-09-13T00:39:37.953 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 157162542~4205502 2024-09-13T00:39:37.962 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: user0 2024-09-13T00:39:38.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:39:38] "GET /metrics HTTP/1.1" 200 38886 "" "Prometheus/2.43.0" 2024-09-13T00:39:39.069 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:38 smithi088 bash[61791]: ts=2024-09-13T00:39:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:39.096 INFO:tasks.workunit.client.0.smithi077.stdout:write: b 95734566~5739482 2024-09-13T00:39:39.145 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 202280492~5822317 2024-09-13T00:39:39.145 INFO:tasks.workunit.client.0.smithi077.stdout:write: b 7935260~1693555 2024-09-13T00:39:39.149 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 2079002~792023 2024-09-13T00:39:39.150 INFO:tasks.workunit.client.0.smithi077.stdout:write: b 134024344~283259 2024-09-13T00:39:39.151 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 64325193~2537616 2024-09-13T00:39:39.151 INFO:tasks.workunit.client.0.smithi077.stdout:write: b 54155002~4848541 2024-09-13T00:39:39.178 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 152046287~7868466 2024-09-13T00:39:39.267 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: user1 2024-09-13T00:39:39.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:39 smithi088 bash[24895]: cluster 2024-09-13T00:39:38.061919+0000 mon.a (mon.0) 87 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2024-09-13T00:39:39.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:39 smithi088 bash[24895]: cluster 2024-09-13T00:39:38.335193+0000 mgr.y (mgr.25304) 37 : cluster [DBG] pgmap v60: 137 pgs: 137 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 838 KiB/s rd, 16 MiB/s wr, 366 op/s 2024-09-13T00:39:39.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:39 smithi077 bash[80681]: cluster 2024-09-13T00:39:38.061919+0000 mon.a (mon.0) 87 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2024-09-13T00:39:39.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:39 smithi077 bash[80681]: cluster 2024-09-13T00:39:38.335193+0000 mgr.y (mgr.25304) 37 : cluster [DBG] pgmap v60: 137 pgs: 137 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 838 KiB/s rd, 16 MiB/s wr, 366 op/s 2024-09-13T00:39:39.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:39 smithi077 bash[32040]: cluster 2024-09-13T00:39:38.061919+0000 mon.a (mon.0) 87 : cluster [DBG] osdmap e327: 8 total, 8 up, 8 in 2024-09-13T00:39:39.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:39 smithi077 bash[32040]: cluster 2024-09-13T00:39:38.335193+0000 mgr.y (mgr.25304) 37 : cluster [DBG] pgmap v60: 137 pgs: 137 active+clean; 1.4 GiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 838 KiB/s rd, 16 MiB/s wr, 366 op/s 2024-09-13T00:39:40.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:40 smithi088 bash[24895]: cluster 2024-09-13T00:39:39.079497+0000 mon.a (mon.0) 88 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2024-09-13T00:39:40.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:40 smithi077 bash[80681]: cluster 2024-09-13T00:39:39.079497+0000 mon.a (mon.0) 88 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2024-09-13T00:39:40.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:40 smithi077 bash[32040]: cluster 2024-09-13T00:39:39.079497+0000 mon.a (mon.0) 88 : cluster [DBG] osdmap e328: 8 total, 8 up, 8 in 2024-09-13T00:39:41.114 INFO:tasks.workunit.client.0.smithi077.stdout:write: c 141829831~7603560 2024-09-13T00:39:41.160 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 68836314~714970 2024-09-13T00:39:41.161 INFO:tasks.workunit.client.0.smithi077.stdout:write: c 172371142~250095 2024-09-13T00:39:41.161 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 48631025~3047559 2024-09-13T00:39:41.162 INFO:tasks.workunit.client.0.smithi077.stdout:write: c 60677470~2526112 2024-09-13T00:39:41.167 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 113517821~8211853 2024-09-13T00:39:41.168 INFO:tasks.workunit.client.0.smithi077.stdout:write: c 202549473~5553336 2024-09-13T00:39:41.235 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 39016212~4082488 2024-09-13T00:39:41.308 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: user2 2024-09-13T00:39:41.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:41 smithi088 bash[24895]: cluster 2024-09-13T00:39:40.088632+0000 mon.a (mon.0) 89 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2024-09-13T00:39:41.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:41 smithi088 bash[24895]: cluster 2024-09-13T00:39:40.335986+0000 mgr.y (mgr.25304) 38 : cluster [DBG] pgmap v63: 137 pgs: 137 active+clean; 1.7 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 37 MiB/s rd, 48 MiB/s wr, 754 op/s 2024-09-13T00:39:41.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:41 smithi077 bash[80681]: cluster 2024-09-13T00:39:40.088632+0000 mon.a (mon.0) 89 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2024-09-13T00:39:41.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:41 smithi077 bash[80681]: cluster 2024-09-13T00:39:40.335986+0000 mgr.y (mgr.25304) 38 : cluster [DBG] pgmap v63: 137 pgs: 137 active+clean; 1.7 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 37 MiB/s rd, 48 MiB/s wr, 754 op/s 2024-09-13T00:39:41.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:41 smithi077 bash[32040]: cluster 2024-09-13T00:39:40.088632+0000 mon.a (mon.0) 89 : cluster [DBG] osdmap e329: 8 total, 8 up, 8 in 2024-09-13T00:39:41.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:41 smithi077 bash[32040]: cluster 2024-09-13T00:39:40.335986+0000 mgr.y (mgr.25304) 38 : cluster [DBG] pgmap v63: 137 pgs: 137 active+clean; 1.7 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 37 MiB/s rd, 48 MiB/s wr, 754 op/s 2024-09-13T00:39:42.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:42 smithi077 bash[80681]: cluster 2024-09-13T00:39:41.098939+0000 mon.a (mon.0) 90 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2024-09-13T00:39:42.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:42 smithi077 bash[32040]: cluster 2024-09-13T00:39:41.098939+0000 mon.a (mon.0) 90 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2024-09-13T00:39:42.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:42 smithi088 bash[24895]: cluster 2024-09-13T00:39:41.098939+0000 mon.a (mon.0) 90 : cluster [DBG] osdmap e330: 8 total, 8 up, 8 in 2024-09-13T00:39:42.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:42 smithi088 bash[61791]: ts=2024-09-13T00:39:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:43.149 INFO:tasks.workunit.client.0.smithi077.stdout:write: d 200702673~7400136 2024-09-13T00:39:43.197 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 39807547~8009585 2024-09-13T00:39:43.197 INFO:tasks.workunit.client.0.smithi077.stdout:write: d 105078052~3438079 2024-09-13T00:39:43.205 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 69206471~3161602 2024-09-13T00:39:43.206 INFO:tasks.workunit.client.0.smithi077.stdout:write: d 104795351~998653 2024-09-13T00:39:43.291 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 147922020~6900920 2024-09-13T00:39:43.293 INFO:tasks.workunit.client.0.smithi077.stdout:write: d 27432709~7449095 2024-09-13T00:39:43.309 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 124113103~8082071 2024-09-13T00:39:43.310 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: user3 2024-09-13T00:39:43.505 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:43 smithi088 bash[24895]: cluster 2024-09-13T00:39:42.100709+0000 mon.a (mon.0) 91 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2024-09-13T00:39:43.506 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:43 smithi088 bash[24895]: cluster 2024-09-13T00:39:42.336776+0000 mgr.y (mgr.25304) 39 : cluster [DBG] pgmap v66: 137 pgs: 137 active+clean; 1.7 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 42 MiB/s rd, 54 MiB/s wr, 849 op/s 2024-09-13T00:39:43.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:43 smithi077 bash[80681]: cluster 2024-09-13T00:39:42.100709+0000 mon.a (mon.0) 91 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2024-09-13T00:39:43.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:43 smithi077 bash[80681]: cluster 2024-09-13T00:39:42.336776+0000 mgr.y (mgr.25304) 39 : cluster [DBG] pgmap v66: 137 pgs: 137 active+clean; 1.7 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 42 MiB/s rd, 54 MiB/s wr, 849 op/s 2024-09-13T00:39:43.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:43 smithi077 bash[32040]: cluster 2024-09-13T00:39:42.100709+0000 mon.a (mon.0) 91 : cluster [DBG] osdmap e331: 8 total, 8 up, 8 in 2024-09-13T00:39:43.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:43 smithi077 bash[32040]: cluster 2024-09-13T00:39:42.336776+0000 mgr.y (mgr.25304) 39 : cluster [DBG] pgmap v66: 137 pgs: 137 active+clean; 1.7 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 42 MiB/s rd, 54 MiB/s wr, 849 op/s 2024-09-13T00:39:43.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:43 smithi088 bash[61791]: ts=2024-09-13T00:39:43.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:44.451 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:44 smithi088 bash[24895]: cluster 2024-09-13T00:39:43.124216+0000 mon.a (mon.0) 92 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2024-09-13T00:39:44.451 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:44 smithi088 bash[24895]: audit 2024-09-13T00:39:43.405351+0000 mon.b (mon.1) 124 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:44.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:44 smithi077 bash[32040]: cluster 2024-09-13T00:39:43.124216+0000 mon.a (mon.0) 92 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2024-09-13T00:39:44.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:44 smithi077 bash[32040]: audit 2024-09-13T00:39:43.405351+0000 mon.b (mon.1) 124 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:44.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:44 smithi077 bash[80681]: cluster 2024-09-13T00:39:43.124216+0000 mon.a (mon.0) 92 : cluster [DBG] osdmap e332: 8 total, 8 up, 8 in 2024-09-13T00:39:44.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:44 smithi077 bash[80681]: audit 2024-09-13T00:39:43.405351+0000 mon.b (mon.1) 124 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:44.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:44 smithi088 bash[61791]: ts=2024-09-13T00:39:44.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:44.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:44 smithi088 bash[61791]: ts=2024-09-13T00:39:44.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:45.186 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 20810280~5202570 2024-09-13T00:39:45.205 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 22110922~2601285 2024-09-13T00:39:45.288 INFO:tasks.workunit.client.0.smithi077.stdout:write: 2 41620561~5202570 2024-09-13T00:39:45.291 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 42921203~2601285 2024-09-13T00:39:45.360 INFO:tasks.workunit.client.0.smithi077.stdout:write: 3 62430842~5202570 2024-09-13T00:39:45.363 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 63731484~2601285 2024-09-13T00:39:45.433 INFO:tasks.workunit.client.0.smithi077.stdout:write: 4 83241123~5202570 2024-09-13T00:39:45.436 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 84541765~2601285 2024-09-13T00:39:45.510 INFO:tasks.workunit.client.0.smithi077.stdout:write: 5 104051404~5202570 2024-09-13T00:39:45.513 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 105352046~2601285 2024-09-13T00:39:45.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:45 smithi077 bash[32040]: cluster 2024-09-13T00:39:44.129983+0000 mon.a (mon.0) 93 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2024-09-13T00:39:45.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:45 smithi077 bash[32040]: cluster 2024-09-13T00:39:44.338166+0000 mgr.y (mgr.25304) 40 : cluster [DBG] pgmap v69: 137 pgs: 137 active+clean; 1.8 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 10 MiB/s rd, 29 MiB/s wr, 376 op/s 2024-09-13T00:39:45.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:45 smithi077 bash[80681]: cluster 2024-09-13T00:39:44.129983+0000 mon.a (mon.0) 93 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2024-09-13T00:39:45.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:45 smithi077 bash[80681]: cluster 2024-09-13T00:39:44.338166+0000 mgr.y (mgr.25304) 40 : cluster [DBG] pgmap v69: 137 pgs: 137 active+clean; 1.8 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 10 MiB/s rd, 29 MiB/s wr, 376 op/s 2024-09-13T00:39:45.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:45 smithi088 bash[24895]: cluster 2024-09-13T00:39:44.129983+0000 mon.a (mon.0) 93 : cluster [DBG] osdmap e333: 8 total, 8 up, 8 in 2024-09-13T00:39:45.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:45 smithi088 bash[24895]: cluster 2024-09-13T00:39:44.338166+0000 mgr.y (mgr.25304) 40 : cluster [DBG] pgmap v69: 137 pgs: 137 active+clean; 1.8 GiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 10 MiB/s rd, 29 MiB/s wr, 376 op/s 2024-09-13T00:39:46.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:46 smithi077 bash[80681]: cluster 2024-09-13T00:39:45.149400+0000 mon.a (mon.0) 94 : cluster [DBG] osdmap e334: 8 total, 8 up, 8 in 2024-09-13T00:39:46.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:46 smithi077 bash[32040]: cluster 2024-09-13T00:39:45.149400+0000 mon.a (mon.0) 94 : cluster [DBG] osdmap e334: 8 total, 8 up, 8 in 2024-09-13T00:39:46.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:46 smithi088 bash[24895]: cluster 2024-09-13T00:39:45.149400+0000 mon.a (mon.0) 94 : cluster [DBG] osdmap e334: 8 total, 8 up, 8 in 2024-09-13T00:39:47.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:47 smithi077 bash[80681]: cluster 2024-09-13T00:39:46.338968+0000 mgr.y (mgr.25304) 41 : cluster [DBG] pgmap v71: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 11 MiB/s rd, 32 MiB/s wr, 366 op/s 2024-09-13T00:39:47.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:47 smithi077 bash[32040]: cluster 2024-09-13T00:39:46.338968+0000 mgr.y (mgr.25304) 41 : cluster [DBG] pgmap v71: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 11 MiB/s rd, 32 MiB/s wr, 366 op/s 2024-09-13T00:39:47.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:47 smithi088 bash[24895]: cluster 2024-09-13T00:39:46.338968+0000 mgr.y (mgr.25304) 41 : cluster [DBG] pgmap v71: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 11 MiB/s rd, 32 MiB/s wr, 366 op/s 2024-09-13T00:39:48.469 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:39:48.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:39:48] "GET /metrics HTTP/1.1" 200 38889 "" "Prometheus/2.43.0" 2024-09-13T00:39:48.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:48 smithi077 bash[32040]: cluster 2024-09-13T00:39:48.341330+0000 mgr.y (mgr.25304) 42 : cluster [DBG] pgmap v72: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 8.0 MiB/s rd, 23 MiB/s wr, 258 op/s 2024-09-13T00:39:48.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:48 smithi077 bash[80681]: cluster 2024-09-13T00:39:48.341330+0000 mgr.y (mgr.25304) 42 : cluster [DBG] pgmap v72: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 8.0 MiB/s rd, 23 MiB/s wr, 258 op/s 2024-09-13T00:39:48.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:48 smithi088 bash[24895]: cluster 2024-09-13T00:39:48.341330+0000 mgr.y (mgr.25304) 42 : cluster [DBG] pgmap v72: 137 pgs: 137 active+clean; 1.8 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 8.0 MiB/s rd, 23 MiB/s wr, 258 op/s 2024-09-13T00:39:48.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:48 smithi088 bash[61791]: ts=2024-09-13T00:39:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:49.005 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:39:50.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:50 smithi077 bash[80681]: cluster 2024-09-13T00:39:50.342933+0000 mgr.y (mgr.25304) 43 : cluster [DBG] pgmap v73: 137 pgs: 137 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s rd, 27 MiB/s wr, 670 op/s 2024-09-13T00:39:50.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:50 smithi077 bash[32040]: cluster 2024-09-13T00:39:50.342933+0000 mgr.y (mgr.25304) 43 : cluster [DBG] pgmap v73: 137 pgs: 137 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s rd, 27 MiB/s wr, 670 op/s 2024-09-13T00:39:50.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:50 smithi088 bash[24895]: cluster 2024-09-13T00:39:50.342933+0000 mgr.y (mgr.25304) 43 : cluster [DBG] pgmap v73: 137 pgs: 137 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 149 MiB/s rd, 27 MiB/s wr, 670 op/s 2024-09-13T00:39:52.031 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.Stress2 (23416 ms) 2024-09-13T00:39:52.032 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMigration.StressLive 2024-09-13T00:39:52.121 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 83886080 2024-09-13T00:39:52.153 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 5310295~1747567 2024-09-13T00:39:52.195 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 71903990~1457926 2024-09-13T00:39:52.195 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 79481511~384891 2024-09-13T00:39:52.213 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 58928392~5466602 2024-09-13T00:39:52.213 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 52962978~2327045 2024-09-13T00:39:52.244 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 65129790~5746881 2024-09-13T00:39:52.245 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 59050983~31155 2024-09-13T00:39:52.258 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 24290359~5421062 2024-09-13T00:39:52.258 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap0 2024-09-13T00:39:52.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:52 smithi088 bash[61791]: ts=2024-09-13T00:39:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:53.353 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 120003127 2024-09-13T00:39:53.380 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 101361666~8244549 2024-09-13T00:39:53.409 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 77052829~149702 2024-09-13T00:39:53.410 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 82548095~8007751 2024-09-13T00:39:53.435 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 65788494~1333106 2024-09-13T00:39:53.435 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 39946668~1255553 2024-09-13T00:39:53.437 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 67248459~3739830 2024-09-13T00:39:53.437 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 40983143~2774136 2024-09-13T00:39:53.494 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 2444032~6021988 2024-09-13T00:39:53.497 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap1 2024-09-13T00:39:53.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:53 smithi088 bash[24895]: cluster 2024-09-13T00:39:52.323870+0000 mon.a (mon.0) 95 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2024-09-13T00:39:53.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:53 smithi088 bash[24895]: cluster 2024-09-13T00:39:52.343886+0000 mgr.y (mgr.25304) 44 : cluster [DBG] pgmap v75: 137 pgs: 137 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 129 MiB/s rd, 10 MiB/s wr, 416 op/s 2024-09-13T00:39:53.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:53 smithi088 bash[61791]: ts=2024-09-13T00:39:53.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:53.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:53 smithi077 bash[80681]: cluster 2024-09-13T00:39:52.323870+0000 mon.a (mon.0) 95 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2024-09-13T00:39:53.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:53 smithi077 bash[80681]: cluster 2024-09-13T00:39:52.343886+0000 mgr.y (mgr.25304) 44 : cluster [DBG] pgmap v75: 137 pgs: 137 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 129 MiB/s rd, 10 MiB/s wr, 416 op/s 2024-09-13T00:39:53.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:53 smithi077 bash[32040]: cluster 2024-09-13T00:39:52.323870+0000 mon.a (mon.0) 95 : cluster [DBG] osdmap e335: 8 total, 8 up, 8 in 2024-09-13T00:39:53.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:53 smithi077 bash[32040]: cluster 2024-09-13T00:39:52.343886+0000 mgr.y (mgr.25304) 44 : cluster [DBG] pgmap v75: 137 pgs: 137 active+clean; 1.9 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 129 MiB/s rd, 10 MiB/s wr, 416 op/s 2024-09-13T00:39:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:54 smithi088 bash[24895]: cluster 2024-09-13T00:39:53.325995+0000 mon.a (mon.0) 96 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2024-09-13T00:39:54.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:54 smithi088 bash[61791]: ts=2024-09-13T00:39:54.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:54.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:54 smithi088 bash[61791]: ts=2024-09-13T00:39:54.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:54.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:54 smithi077 bash[80681]: cluster 2024-09-13T00:39:53.325995+0000 mon.a (mon.0) 96 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2024-09-13T00:39:54.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:54 smithi077 bash[32040]: cluster 2024-09-13T00:39:53.325995+0000 mon.a (mon.0) 96 : cluster [DBG] osdmap e336: 8 total, 8 up, 8 in 2024-09-13T00:39:55.379 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap1 2024-09-13T00:39:55.410 INFO:tasks.workunit.client.0.smithi077.stdout:clone image175 -> image178 2024-09-13T00:39:55.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:55 smithi088 bash[24895]: cluster 2024-09-13T00:39:54.339031+0000 mon.a (mon.0) 97 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2024-09-13T00:39:55.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:55 smithi088 bash[24895]: cluster 2024-09-13T00:39:54.344953+0000 mgr.y (mgr.25304) 45 : cluster [DBG] pgmap v78: 137 pgs: 137 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 336 MiB/s rd, 18 MiB/s wr, 719 op/s 2024-09-13T00:39:55.667 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 85510395 2024-09-13T00:39:55.751 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 32892985~1254545 2024-09-13T00:39:55.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:55 smithi077 bash[80681]: cluster 2024-09-13T00:39:54.339031+0000 mon.a (mon.0) 97 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2024-09-13T00:39:55.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:55 smithi077 bash[80681]: cluster 2024-09-13T00:39:54.344953+0000 mgr.y (mgr.25304) 45 : cluster [DBG] pgmap v78: 137 pgs: 137 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 336 MiB/s rd, 18 MiB/s wr, 719 op/s 2024-09-13T00:39:55.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:55 smithi077 bash[32040]: cluster 2024-09-13T00:39:54.339031+0000 mon.a (mon.0) 97 : cluster [DBG] osdmap e337: 8 total, 8 up, 8 in 2024-09-13T00:39:55.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:55 smithi077 bash[32040]: cluster 2024-09-13T00:39:54.344953+0000 mgr.y (mgr.25304) 45 : cluster [DBG] pgmap v78: 137 pgs: 137 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 336 MiB/s rd, 18 MiB/s wr, 719 op/s 2024-09-13T00:39:55.788 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 24045675~7278587 2024-09-13T00:39:55.805 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 31652350~1965391 2024-09-13T00:39:55.831 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 15218878~7332153 2024-09-13T00:39:55.847 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 20256260~6434278 2024-09-13T00:39:55.912 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 30357904~4222458 2024-09-13T00:39:55.919 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 77002973~7139798 2024-09-13T00:39:55.983 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 55878510~757987 2024-09-13T00:39:56.007 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap2 2024-09-13T00:39:56.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:56 smithi077 bash[80681]: cluster 2024-09-13T00:39:55.359728+0000 mon.a (mon.0) 98 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2024-09-13T00:39:56.848 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:56 smithi077 bash[32040]: cluster 2024-09-13T00:39:55.359728+0000 mon.a (mon.0) 98 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2024-09-13T00:39:56.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:56 smithi088 bash[24895]: cluster 2024-09-13T00:39:55.359728+0000 mon.a (mon.0) 98 : cluster [DBG] osdmap e338: 8 total, 8 up, 8 in 2024-09-13T00:39:57.398 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 55275901~938891 2024-09-13T00:39:57.418 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 50116610~675556 2024-09-13T00:39:57.419 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 50965354~6258669 2024-09-13T00:39:57.438 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 44791381~4739597 2024-09-13T00:39:57.439 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 71296148~2872998 2024-09-13T00:39:57.442 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 2309920~7966596 2024-09-13T00:39:57.443 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 32762650~178900 2024-09-13T00:39:57.443 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 66444776~46275 2024-09-13T00:39:57.444 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: snap3 2024-09-13T00:39:57.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:57 smithi077 bash[80681]: cluster 2024-09-13T00:39:56.345710+0000 mgr.y (mgr.25304) 46 : cluster [DBG] pgmap v80: 137 pgs: 137 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 247 MiB/s rd, 12 MiB/s wr, 257 op/s 2024-09-13T00:39:57.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:57 smithi077 bash[80681]: cluster 2024-09-13T00:39:56.368214+0000 mon.a (mon.0) 99 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2024-09-13T00:39:57.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:57 smithi077 bash[32040]: cluster 2024-09-13T00:39:56.345710+0000 mgr.y (mgr.25304) 46 : cluster [DBG] pgmap v80: 137 pgs: 137 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 247 MiB/s rd, 12 MiB/s wr, 257 op/s 2024-09-13T00:39:57.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:57 smithi077 bash[32040]: cluster 2024-09-13T00:39:56.368214+0000 mon.a (mon.0) 99 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2024-09-13T00:39:57.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:57 smithi088 bash[24895]: cluster 2024-09-13T00:39:56.345710+0000 mgr.y (mgr.25304) 46 : cluster [DBG] pgmap v80: 137 pgs: 137 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 247 MiB/s rd, 12 MiB/s wr, 257 op/s 2024-09-13T00:39:57.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:57 smithi088 bash[24895]: cluster 2024-09-13T00:39:56.368214+0000 mon.a (mon.0) 99 : cluster [DBG] osdmap e339: 8 total, 8 up, 8 in 2024-09-13T00:39:58.659 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:58 smithi088 bash[24895]: cluster 2024-09-13T00:39:57.369756+0000 mon.a (mon.0) 100 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2024-09-13T00:39:58.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:58 smithi077 bash[80681]: cluster 2024-09-13T00:39:57.369756+0000 mon.a (mon.0) 100 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2024-09-13T00:39:58.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:39:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:39:58] "GET /metrics HTTP/1.1" 200 38879 "" "Prometheus/2.43.0" 2024-09-13T00:39:58.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:58 smithi077 bash[32040]: cluster 2024-09-13T00:39:57.369756+0000 mon.a (mon.0) 100 : cluster [DBG] osdmap e340: 8 total, 8 up, 8 in 2024-09-13T00:39:59.101 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:39:58 smithi088 bash[61791]: ts=2024-09-13T00:39:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:39:59.421 INFO:tasks.workunit.client.0.smithi077.stdout:snap_protect: snap3 2024-09-13T00:39:59.457 INFO:tasks.workunit.client.0.smithi077.stdout:clone image178 -> image180 2024-09-13T00:39:59.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:59 smithi077 bash[80681]: cluster 2024-09-13T00:39:58.346447+0000 mgr.y (mgr.25304) 47 : cluster [DBG] pgmap v83: 137 pgs: 137 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 35 MiB/s rd, 4.6 MiB/s wr, 96 op/s 2024-09-13T00:39:59.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:59 smithi077 bash[80681]: cluster 2024-09-13T00:39:58.390386+0000 mon.a (mon.0) 101 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2024-09-13T00:39:59.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:39:59 smithi077 bash[80681]: audit 2024-09-13T00:39:58.407828+0000 mon.b (mon.1) 125 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:59.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:59 smithi077 bash[32040]: cluster 2024-09-13T00:39:58.346447+0000 mgr.y (mgr.25304) 47 : cluster [DBG] pgmap v83: 137 pgs: 137 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 35 MiB/s rd, 4.6 MiB/s wr, 96 op/s 2024-09-13T00:39:59.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:59 smithi077 bash[32040]: cluster 2024-09-13T00:39:58.390386+0000 mon.a (mon.0) 101 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2024-09-13T00:39:59.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:39:59 smithi077 bash[32040]: audit 2024-09-13T00:39:58.407828+0000 mon.b (mon.1) 125 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:39:59.810 INFO:tasks.workunit.client.0.smithi077.stdout:migration_prepare 2024-09-13T00:39:59.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:59 smithi088 bash[24895]: cluster 2024-09-13T00:39:58.346447+0000 mgr.y (mgr.25304) 47 : cluster [DBG] pgmap v83: 137 pgs: 137 active+clean; 2.0 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 35 MiB/s rd, 4.6 MiB/s wr, 96 op/s 2024-09-13T00:39:59.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:59 smithi088 bash[24895]: cluster 2024-09-13T00:39:58.390386+0000 mon.a (mon.0) 101 : cluster [DBG] osdmap e341: 8 total, 8 up, 8 in 2024-09-13T00:39:59.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:39:59 smithi088 bash[24895]: audit 2024-09-13T00:39:58.407828+0000 mon.b (mon.1) 125 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:00.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: debug 2024-09-13T00:39:59.996+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable; 1 pool(s) do not have an application enabled 2024-09-13T00:40:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: debug 2024-09-13T00:39:59.996+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:40:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: debug 2024-09-13T00:39:59.996+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: debug 2024-09-13T00:39:59.996+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: debug 2024-09-13T00:39:59.996+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: debug 2024-09-13T00:39:59.996+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: debug 2024-09-13T00:39:59.996+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: debug 2024-09-13T00:39:59.996+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: debug 2024-09-13T00:39:59.996+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: debug 2024-09-13T00:39:59.996+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: debug 2024-09-13T00:39:59.996+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T00:40:00.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: debug 2024-09-13T00:39:59.996+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : application not enabled on pool 'test-librbd-smithi077-68798-8' 2024-09-13T00:40:00.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: debug 2024-09-13T00:39:59.996+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T00:40:00.647 INFO:tasks.workunit.client.0.smithi077.stdout:migration_execute 2024-09-13T00:40:00.647 INFO:tasks.workunit.client.0.smithi077.stdout:write: a 5687255~6378621 2024-09-13T00:40:00.739 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 26341485~1009511 2024-09-13T00:40:00.776 INFO:tasks.workunit.client.0.smithi077.stdout:write: a 59079280~5672754 2024-09-13T00:40:00.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[32040]: cluster 2024-09-13T00:39:59.391891+0000 mon.a (mon.0) 102 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2024-09-13T00:40:00.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[32040]: cluster 2024-09-13T00:39:59.405382+0000 mon.a (mon.0) 103 : cluster [DBG] mgrmap e38: y(active, since 61s), standbys: x 2024-09-13T00:40:00.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[32040]: cluster 2024-09-13T00:40:00.000166+0000 mon.a (mon.0) 104 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable; 1 pool(s) do not have an application enabled 2024-09-13T00:40:00.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[32040]: cluster 2024-09-13T00:40:00.000229+0000 mon.a (mon.0) 105 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:40:00.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[32040]: cluster 2024-09-13T00:40:00.000275+0000 mon.a (mon.0) 106 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[32040]: cluster 2024-09-13T00:40:00.000317+0000 mon.a (mon.0) 107 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[32040]: cluster 2024-09-13T00:40:00.000362+0000 mon.a (mon.0) 108 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[32040]: cluster 2024-09-13T00:40:00.000390+0000 mon.a (mon.0) 109 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[32040]: cluster 2024-09-13T00:40:00.000424+0000 mon.a (mon.0) 110 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[32040]: cluster 2024-09-13T00:40:00.000476+0000 mon.a (mon.0) 111 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[32040]: cluster 2024-09-13T00:40:00.000505+0000 mon.a (mon.0) 112 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[32040]: cluster 2024-09-13T00:40:00.000535+0000 mon.a (mon.0) 113 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[32040]: cluster 2024-09-13T00:40:00.000576+0000 mon.a (mon.0) 114 : cluster [ERR] [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T00:40:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[32040]: cluster 2024-09-13T00:40:00.000628+0000 mon.a (mon.0) 115 : cluster [ERR] application not enabled on pool 'test-librbd-smithi077-68798-8' 2024-09-13T00:40:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[32040]: cluster 2024-09-13T00:40:00.000666+0000 mon.a (mon.0) 116 : cluster [ERR] use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T00:40:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[32040]: cluster 2024-09-13T00:40:00.347308+0000 mgr.y (mgr.25304) 48 : cluster [DBG] pgmap v86: 137 pgs: 137 active+clean; 2.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 4.3 MiB/s rd, 16 MiB/s wr, 295 op/s 2024-09-13T00:40:00.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: cluster 2024-09-13T00:39:59.391891+0000 mon.a (mon.0) 102 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2024-09-13T00:40:00.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: cluster 2024-09-13T00:39:59.405382+0000 mon.a (mon.0) 103 : cluster [DBG] mgrmap e38: y(active, since 61s), standbys: x 2024-09-13T00:40:00.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: cluster 2024-09-13T00:40:00.000166+0000 mon.a (mon.0) 104 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable; 1 pool(s) do not have an application enabled 2024-09-13T00:40:00.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: cluster 2024-09-13T00:40:00.000229+0000 mon.a (mon.0) 105 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:40:00.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: cluster 2024-09-13T00:40:00.000275+0000 mon.a (mon.0) 106 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: cluster 2024-09-13T00:40:00.000317+0000 mon.a (mon.0) 107 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: cluster 2024-09-13T00:40:00.000362+0000 mon.a (mon.0) 108 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: cluster 2024-09-13T00:40:00.000390+0000 mon.a (mon.0) 109 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: cluster 2024-09-13T00:40:00.000424+0000 mon.a (mon.0) 110 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: cluster 2024-09-13T00:40:00.000476+0000 mon.a (mon.0) 111 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: cluster 2024-09-13T00:40:00.000505+0000 mon.a (mon.0) 112 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: cluster 2024-09-13T00:40:00.000535+0000 mon.a (mon.0) 113 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: cluster 2024-09-13T00:40:00.000576+0000 mon.a (mon.0) 114 : cluster [ERR] [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T00:40:00.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: cluster 2024-09-13T00:40:00.000628+0000 mon.a (mon.0) 115 : cluster [ERR] application not enabled on pool 'test-librbd-smithi077-68798-8' 2024-09-13T00:40:00.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: cluster 2024-09-13T00:40:00.000666+0000 mon.a (mon.0) 116 : cluster [ERR] use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T00:40:00.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:00 smithi077 bash[80681]: cluster 2024-09-13T00:40:00.347308+0000 mgr.y (mgr.25304) 48 : cluster [DBG] pgmap v86: 137 pgs: 137 active+clean; 2.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 4.3 MiB/s rd, 16 MiB/s wr, 295 op/s 2024-09-13T00:40:00.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:00 smithi088 bash[24895]: cluster 2024-09-13T00:39:59.391891+0000 mon.a (mon.0) 102 : cluster [DBG] osdmap e342: 8 total, 8 up, 8 in 2024-09-13T00:40:00.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:00 smithi088 bash[24895]: cluster 2024-09-13T00:39:59.405382+0000 mon.a (mon.0) 103 : cluster [DBG] mgrmap e38: y(active, since 61s), standbys: x 2024-09-13T00:40:00.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:00 smithi088 bash[24895]: cluster 2024-09-13T00:40:00.000166+0000 mon.a (mon.0) 104 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable; 1 pool(s) do not have an application enabled 2024-09-13T00:40:00.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:00 smithi088 bash[24895]: cluster 2024-09-13T00:40:00.000229+0000 mon.a (mon.0) 105 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:40:00.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:00 smithi088 bash[24895]: cluster 2024-09-13T00:40:00.000275+0000 mon.a (mon.0) 106 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:00 smithi088 bash[24895]: cluster 2024-09-13T00:40:00.000317+0000 mon.a (mon.0) 107 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:00 smithi088 bash[24895]: cluster 2024-09-13T00:40:00.000362+0000 mon.a (mon.0) 108 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:00 smithi088 bash[24895]: cluster 2024-09-13T00:40:00.000390+0000 mon.a (mon.0) 109 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:00 smithi088 bash[24895]: cluster 2024-09-13T00:40:00.000424+0000 mon.a (mon.0) 110 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:00 smithi088 bash[24895]: cluster 2024-09-13T00:40:00.000476+0000 mon.a (mon.0) 111 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:00 smithi088 bash[24895]: cluster 2024-09-13T00:40:00.000505+0000 mon.a (mon.0) 112 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:00 smithi088 bash[24895]: cluster 2024-09-13T00:40:00.000535+0000 mon.a (mon.0) 113 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:00.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:00 smithi088 bash[24895]: cluster 2024-09-13T00:40:00.000576+0000 mon.a (mon.0) 114 : cluster [ERR] [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-13T00:40:00.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:00 smithi088 bash[24895]: cluster 2024-09-13T00:40:00.000628+0000 mon.a (mon.0) 115 : cluster [ERR] application not enabled on pool 'test-librbd-smithi077-68798-8' 2024-09-13T00:40:00.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:00 smithi088 bash[24895]: cluster 2024-09-13T00:40:00.000666+0000 mon.a (mon.0) 116 : cluster [ERR] use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-13T00:40:00.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:00 smithi088 bash[24895]: cluster 2024-09-13T00:40:00.347308+0000 mgr.y (mgr.25304) 48 : cluster [DBG] pgmap v86: 137 pgs: 137 active+clean; 2.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 4.3 MiB/s rd, 16 MiB/s wr, 295 op/s 2024-09-13T00:40:00.863 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 62588017~5032576 2024-09-13T00:40:00.899 INFO:tasks.workunit.client.0.smithi077.stdout:write: a 3076000~1063426 2024-09-13T00:40:00.936 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 30335492~4029729 2024-09-13T00:40:00.982 INFO:tasks.workunit.client.0.smithi077.stdout:write: a 80299829~5210566 2024-09-13T00:40:01.057 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 14274730~7421474 2024-09-13T00:40:01.093 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: user0 2024-09-13T00:40:02.451 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 101615952 2024-09-13T00:40:02.480 INFO:tasks.workunit.client.0.smithi077.stdout:write: b 2914478~2737132 2024-09-13T00:40:02.500 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 17624067~3823115 2024-09-13T00:40:02.500 INFO:tasks.workunit.client.0.smithi077.stdout:write: b 20178319~7205321 2024-09-13T00:40:02.528 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 73169837~1711941 2024-09-13T00:40:02.532 INFO:tasks.workunit.client.0.smithi077.stdout:write: b 3478232~4233637 2024-09-13T00:40:02.536 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 95162852~6453100 2024-09-13T00:40:02.538 INFO:tasks.workunit.client.0.smithi077.stdout:write: b 96717802~3769882 2024-09-13T00:40:02.551 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 94196113~2067885 2024-09-13T00:40:02.551 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: user1 2024-09-13T00:40:02.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:02 smithi088 bash[24895]: cluster 2024-09-13T00:40:01.416520+0000 mon.a (mon.0) 117 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2024-09-13T00:40:02.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:02 smithi088 bash[24895]: cluster 2024-09-13T00:40:02.348036+0000 mgr.y (mgr.25304) 49 : cluster [DBG] pgmap v88: 137 pgs: 137 active+clean; 2.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3.4 MiB/s rd, 13 MiB/s wr, 237 op/s 2024-09-13T00:40:02.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:02 smithi088 bash[61791]: ts=2024-09-13T00:40:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:02.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:02 smithi077 bash[80681]: cluster 2024-09-13T00:40:01.416520+0000 mon.a (mon.0) 117 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2024-09-13T00:40:02.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:02 smithi077 bash[80681]: cluster 2024-09-13T00:40:02.348036+0000 mgr.y (mgr.25304) 49 : cluster [DBG] pgmap v88: 137 pgs: 137 active+clean; 2.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3.4 MiB/s rd, 13 MiB/s wr, 237 op/s 2024-09-13T00:40:02.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:02 smithi077 bash[32040]: cluster 2024-09-13T00:40:01.416520+0000 mon.a (mon.0) 117 : cluster [DBG] osdmap e343: 8 total, 8 up, 8 in 2024-09-13T00:40:02.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:02 smithi077 bash[32040]: cluster 2024-09-13T00:40:02.348036+0000 mgr.y (mgr.25304) 49 : cluster [DBG] pgmap v88: 137 pgs: 137 active+clean; 2.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 3.4 MiB/s rd, 13 MiB/s wr, 237 op/s 2024-09-13T00:40:03.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:03 smithi077 bash[80681]: cluster 2024-09-13T00:40:02.429208+0000 mon.a (mon.0) 118 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2024-09-13T00:40:03.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:03 smithi077 bash[32040]: cluster 2024-09-13T00:40:02.429208+0000 mon.a (mon.0) 118 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2024-09-13T00:40:03.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:03 smithi088 bash[24895]: cluster 2024-09-13T00:40:02.429208+0000 mon.a (mon.0) 118 : cluster [DBG] osdmap e344: 8 total, 8 up, 8 in 2024-09-13T00:40:03.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:03 smithi088 bash[61791]: ts=2024-09-13T00:40:03.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:04.474 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 175231802 2024-09-13T00:40:04.504 INFO:tasks.workunit.client.0.smithi077.stdout:write: c 165823892~1622754 2024-09-13T00:40:04.531 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 30610988~3775975 2024-09-13T00:40:04.531 INFO:tasks.workunit.client.0.smithi077.stdout:write: c 172318472~2913330 2024-09-13T00:40:04.540 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 90124130~7588815 2024-09-13T00:40:04.541 INFO:tasks.workunit.client.0.smithi077.stdout:write: c 6621681~4385717 2024-09-13T00:40:04.555 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 121300668~5621411 2024-09-13T00:40:04.556 INFO:tasks.workunit.client.0.smithi077.stdout:write: c 152794897~3524609 2024-09-13T00:40:04.566 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 117300680~6216260 2024-09-13T00:40:04.567 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: user2 2024-09-13T00:40:04.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:04 smithi077 bash[80681]: cluster 2024-09-13T00:40:03.428497+0000 mon.a (mon.0) 119 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2024-09-13T00:40:04.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:04 smithi077 bash[80681]: cluster 2024-09-13T00:40:04.349498+0000 mgr.y (mgr.25304) 50 : cluster [DBG] pgmap v91: 137 pgs: 137 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 17 MiB/s rd, 24 MiB/s wr, 586 op/s 2024-09-13T00:40:04.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:04 smithi077 bash[32040]: cluster 2024-09-13T00:40:03.428497+0000 mon.a (mon.0) 119 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2024-09-13T00:40:04.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:04 smithi077 bash[32040]: cluster 2024-09-13T00:40:04.349498+0000 mgr.y (mgr.25304) 50 : cluster [DBG] pgmap v91: 137 pgs: 137 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 17 MiB/s rd, 24 MiB/s wr, 586 op/s 2024-09-13T00:40:04.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:04 smithi088 bash[24895]: cluster 2024-09-13T00:40:03.428497+0000 mon.a (mon.0) 119 : cluster [DBG] osdmap e345: 8 total, 8 up, 8 in 2024-09-13T00:40:04.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:04 smithi088 bash[24895]: cluster 2024-09-13T00:40:04.349498+0000 mgr.y (mgr.25304) 50 : cluster [DBG] pgmap v91: 137 pgs: 137 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 17 MiB/s rd, 24 MiB/s wr, 586 op/s 2024-09-13T00:40:04.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:04 smithi088 bash[61791]: ts=2024-09-13T00:40:04.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:04.852 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:04 smithi088 bash[61791]: ts=2024-09-13T00:40:04.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:05.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:05 smithi077 bash[80681]: cluster 2024-09-13T00:40:04.454602+0000 mon.a (mon.0) 120 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2024-09-13T00:40:05.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:05 smithi077 bash[32040]: cluster 2024-09-13T00:40:04.454602+0000 mon.a (mon.0) 120 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2024-09-13T00:40:05.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:05 smithi088 bash[24895]: cluster 2024-09-13T00:40:04.454602+0000 mon.a (mon.0) 120 : cluster [DBG] osdmap e346: 8 total, 8 up, 8 in 2024-09-13T00:40:06.492 INFO:tasks.workunit.client.0.smithi077.stdout:write: d 137478999~5032974 2024-09-13T00:40:06.523 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 20805803~7292007 2024-09-13T00:40:06.524 INFO:tasks.workunit.client.0.smithi077.stdout:write: d 33259167~6593856 2024-09-13T00:40:06.551 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 63391391~2093080 2024-09-13T00:40:06.552 INFO:tasks.workunit.client.0.smithi077.stdout:write: d 117446030~4431497 2024-09-13T00:40:06.558 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 61400470~3952122 2024-09-13T00:40:06.599 INFO:tasks.workunit.client.0.smithi077.stdout:write: d 38716517~7014991 2024-09-13T00:40:06.627 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 51654593~2445036 2024-09-13T00:40:06.628 INFO:tasks.workunit.client.0.smithi077.stdout:snap_create: user3 2024-09-13T00:40:06.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:06 smithi077 bash[80681]: cluster 2024-09-13T00:40:05.455723+0000 mon.a (mon.0) 121 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2024-09-13T00:40:06.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:06 smithi077 bash[80681]: cluster 2024-09-13T00:40:06.350217+0000 mgr.y (mgr.25304) 51 : cluster [DBG] pgmap v94: 137 pgs: 137 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 21 MiB/s rd, 30 MiB/s wr, 727 op/s 2024-09-13T00:40:06.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:06 smithi077 bash[32040]: cluster 2024-09-13T00:40:05.455723+0000 mon.a (mon.0) 121 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2024-09-13T00:40:06.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:06 smithi077 bash[32040]: cluster 2024-09-13T00:40:06.350217+0000 mgr.y (mgr.25304) 51 : cluster [DBG] pgmap v94: 137 pgs: 137 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 21 MiB/s rd, 30 MiB/s wr, 727 op/s 2024-09-13T00:40:06.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:06 smithi088 bash[24895]: cluster 2024-09-13T00:40:05.455723+0000 mon.a (mon.0) 121 : cluster [DBG] osdmap e347: 8 total, 8 up, 8 in 2024-09-13T00:40:06.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:06 smithi088 bash[24895]: cluster 2024-09-13T00:40:06.350217+0000 mgr.y (mgr.25304) 51 : cluster [DBG] pgmap v94: 137 pgs: 137 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 21 MiB/s rd, 30 MiB/s wr, 727 op/s 2024-09-13T00:40:07.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:07 smithi077 bash[80681]: cluster 2024-09-13T00:40:06.476338+0000 mon.a (mon.0) 122 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2024-09-13T00:40:07.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:07 smithi077 bash[32040]: cluster 2024-09-13T00:40:06.476338+0000 mon.a (mon.0) 122 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2024-09-13T00:40:07.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:07 smithi088 bash[24895]: cluster 2024-09-13T00:40:06.476338+0000 mon.a (mon.0) 122 : cluster [DBG] osdmap e348: 8 total, 8 up, 8 in 2024-09-13T00:40:08.540 INFO:tasks.workunit.client.0.smithi077.stdout:write: 1 17523180~4380795 2024-09-13T00:40:08.557 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 18618378~2190397 2024-09-13T00:40:08.593 INFO:tasks.workunit.client.0.smithi077.stdout:write: 2 35046360~4380795 2024-09-13T00:40:08.595 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 36141558~2190397 2024-09-13T00:40:08.625 INFO:tasks.workunit.client.0.smithi077.stdout:write: 3 52569540~4380795 2024-09-13T00:40:08.628 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 53664738~2190397 2024-09-13T00:40:08.675 INFO:tasks.workunit.client.0.smithi077.stdout:write: 4 70092720~4380795 2024-09-13T00:40:08.677 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 71187918~2190397 2024-09-13T00:40:08.706 INFO:tasks.workunit.client.0.smithi077.stdout:write: 5 87615901~4380795 2024-09-13T00:40:08.708 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 88711099~2190397 2024-09-13T00:40:08.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:40:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:40:08] "GET /metrics HTTP/1.1" 200 38879 "" "Prometheus/2.43.0" 2024-09-13T00:40:08.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:08 smithi077 bash[32040]: cluster 2024-09-13T00:40:07.476937+0000 mon.a (mon.0) 123 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2024-09-13T00:40:08.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:08 smithi077 bash[32040]: cluster 2024-09-13T00:40:08.350956+0000 mgr.y (mgr.25304) 52 : cluster [DBG] pgmap v97: 137 pgs: 137 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2024-09-13T00:40:08.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:08 smithi077 bash[80681]: cluster 2024-09-13T00:40:07.476937+0000 mon.a (mon.0) 123 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2024-09-13T00:40:08.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:08 smithi077 bash[80681]: cluster 2024-09-13T00:40:08.350956+0000 mgr.y (mgr.25304) 52 : cluster [DBG] pgmap v97: 137 pgs: 137 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2024-09-13T00:40:08.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:08 smithi088 bash[24895]: cluster 2024-09-13T00:40:07.476937+0000 mon.a (mon.0) 123 : cluster [DBG] osdmap e349: 8 total, 8 up, 8 in 2024-09-13T00:40:08.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:08 smithi088 bash[24895]: cluster 2024-09-13T00:40:08.350956+0000 mgr.y (mgr.25304) 52 : cluster [DBG] pgmap v97: 137 pgs: 137 active+clean; 2.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail 2024-09-13T00:40:08.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:08 smithi088 bash[61791]: ts=2024-09-13T00:40:08.663Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:08.910 INFO:tasks.workunit.client.0.smithi077.stdout:migration_commit 2024-09-13T00:40:09.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:09 smithi077 bash[80681]: cluster 2024-09-13T00:40:08.499314+0000 mon.a (mon.0) 124 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2024-09-13T00:40:09.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:09 smithi077 bash[32040]: cluster 2024-09-13T00:40:08.499314+0000 mon.a (mon.0) 124 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2024-09-13T00:40:09.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:09 smithi088 bash[24895]: cluster 2024-09-13T00:40:08.499314+0000 mon.a (mon.0) 124 : cluster [DBG] osdmap e350: 8 total, 8 up, 8 in 2024-09-13T00:40:10.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:10 smithi077 bash[80681]: cluster 2024-09-13T00:40:10.352562+0000 mgr.y (mgr.25304) 53 : cluster [DBG] pgmap v99: 137 pgs: 137 active+clean; 2.3 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 27 MiB/s wr, 443 op/s 2024-09-13T00:40:10.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:10 smithi077 bash[32040]: cluster 2024-09-13T00:40:10.352562+0000 mgr.y (mgr.25304) 53 : cluster [DBG] pgmap v99: 137 pgs: 137 active+clean; 2.3 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 27 MiB/s wr, 443 op/s 2024-09-13T00:40:10.826 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMigration.StressLive (18794 ms) 2024-09-13T00:40:10.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:10 smithi088 bash[24895]: cluster 2024-09-13T00:40:10.352562+0000 mgr.y (mgr.25304) 53 : cluster [DBG] pgmap v99: 137 pgs: 137 active+clean; 2.3 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 14 MiB/s rd, 27 MiB/s wr, 443 op/s 2024-09-13T00:40:12.530 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 57 tests from TestMigration (292175 ms total) 2024-09-13T00:40:12.531 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:40:12.531 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 6 tests from TestGroup 2024-09-13T00:40:12.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:12 smithi088 bash[24895]: cluster 2024-09-13T00:40:11.513653+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2024-09-13T00:40:12.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:12 smithi088 bash[24895]: cluster 2024-09-13T00:40:12.353350+0000 mgr.y (mgr.25304) 54 : cluster [DBG] pgmap v101: 105 pgs: 105 active+clean; 2.2 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 6.3 MiB/s wr, 6 op/s 2024-09-13T00:40:12.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:12 smithi088 bash[61791]: ts=2024-09-13T00:40:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:12.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:12 smithi077 bash[80681]: cluster 2024-09-13T00:40:11.513653+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2024-09-13T00:40:12.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:12 smithi077 bash[80681]: cluster 2024-09-13T00:40:12.353350+0000 mgr.y (mgr.25304) 54 : cluster [DBG] pgmap v101: 105 pgs: 105 active+clean; 2.2 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 6.3 MiB/s wr, 6 op/s 2024-09-13T00:40:12.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:12 smithi077 bash[32040]: cluster 2024-09-13T00:40:11.513653+0000 mon.a (mon.0) 125 : cluster [DBG] osdmap e351: 8 total, 8 up, 8 in 2024-09-13T00:40:12.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:12 smithi077 bash[32040]: cluster 2024-09-13T00:40:12.353350+0000 mgr.y (mgr.25304) 54 : cluster [DBG] pgmap v101: 105 pgs: 105 active+clean; 2.2 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 6.3 MiB/s wr, 6 op/s 2024-09-13T00:40:13.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:13 smithi088 bash[61791]: ts=2024-09-13T00:40:13.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:13.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:13 smithi088 bash[24895]: cluster 2024-09-13T00:40:12.505743+0000 mon.a (mon.0) 126 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:40:13.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:13 smithi088 bash[24895]: cluster 2024-09-13T00:40:12.522318+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2024-09-13T00:40:13.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:13 smithi088 bash[24895]: audit 2024-09-13T00:40:13.419567+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:13.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:13 smithi088 bash[24895]: audit 2024-09-13T00:40:13.421035+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:14.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:13 smithi077 bash[80681]: cluster 2024-09-13T00:40:12.505743+0000 mon.a (mon.0) 126 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:40:14.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:13 smithi077 bash[80681]: cluster 2024-09-13T00:40:12.522318+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2024-09-13T00:40:14.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:13 smithi077 bash[80681]: audit 2024-09-13T00:40:13.419567+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:14.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:13 smithi077 bash[80681]: audit 2024-09-13T00:40:13.421035+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:14.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:13 smithi077 bash[32040]: cluster 2024-09-13T00:40:12.505743+0000 mon.a (mon.0) 126 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:40:14.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:13 smithi077 bash[32040]: cluster 2024-09-13T00:40:12.522318+0000 mon.a (mon.0) 127 : cluster [DBG] osdmap e352: 8 total, 8 up, 8 in 2024-09-13T00:40:14.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:13 smithi077 bash[32040]: audit 2024-09-13T00:40:13.419567+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:14.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:13 smithi077 bash[32040]: audit 2024-09-13T00:40:13.421035+0000 mon.b (mon.1) 126 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:14.530 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestGroup.group_create 2024-09-13T00:40:14.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:14 smithi088 bash[61791]: ts=2024-09-13T00:40:14.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:14.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:14 smithi088 bash[61791]: ts=2024-09-13T00:40:14.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:14.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:14 smithi088 bash[24895]: cluster 2024-09-13T00:40:13.525949+0000 mon.a (mon.0) 129 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2024-09-13T00:40:14.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:14 smithi088 bash[24895]: audit 2024-09-13T00:40:13.531409+0000 mon.c (mon.2) 236 : audit [INF] from='client.? 172.21.15.77:0/3571885628' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:14.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:14 smithi088 bash[24895]: audit 2024-09-13T00:40:13.533280+0000 mon.a (mon.0) 130 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:14.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:14 smithi088 bash[24895]: cluster 2024-09-13T00:40:14.354641+0000 mgr.y (mgr.25304) 55 : cluster [DBG] pgmap v104: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:14.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:14 smithi088 bash[24895]: audit 2024-09-13T00:40:14.531193+0000 mon.a (mon.0) 131 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:40:14.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:14 smithi088 bash[24895]: cluster 2024-09-13T00:40:14.531746+0000 mon.a (mon.0) 132 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2024-09-13T00:40:15.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:14 smithi077 bash[80681]: cluster 2024-09-13T00:40:13.525949+0000 mon.a (mon.0) 129 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2024-09-13T00:40:15.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:14 smithi077 bash[80681]: audit 2024-09-13T00:40:13.531409+0000 mon.c (mon.2) 236 : audit [INF] from='client.? 172.21.15.77:0/3571885628' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:15.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:14 smithi077 bash[80681]: audit 2024-09-13T00:40:13.533280+0000 mon.a (mon.0) 130 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:15.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:14 smithi077 bash[80681]: cluster 2024-09-13T00:40:14.354641+0000 mgr.y (mgr.25304) 55 : cluster [DBG] pgmap v104: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:15.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:14 smithi077 bash[80681]: audit 2024-09-13T00:40:14.531193+0000 mon.a (mon.0) 131 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:40:15.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:14 smithi077 bash[80681]: cluster 2024-09-13T00:40:14.531746+0000 mon.a (mon.0) 132 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2024-09-13T00:40:15.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:14 smithi077 bash[32040]: cluster 2024-09-13T00:40:13.525949+0000 mon.a (mon.0) 129 : cluster [DBG] osdmap e353: 8 total, 8 up, 8 in 2024-09-13T00:40:15.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:14 smithi077 bash[32040]: audit 2024-09-13T00:40:13.531409+0000 mon.c (mon.2) 236 : audit [INF] from='client.? 172.21.15.77:0/3571885628' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:15.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:14 smithi077 bash[32040]: audit 2024-09-13T00:40:13.533280+0000 mon.a (mon.0) 130 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-9","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:15.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:14 smithi077 bash[32040]: cluster 2024-09-13T00:40:14.354641+0000 mgr.y (mgr.25304) 55 : cluster [DBG] pgmap v104: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:15.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:14 smithi077 bash[32040]: audit 2024-09-13T00:40:14.531193+0000 mon.a (mon.0) 131 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-9","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:40:15.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:14 smithi077 bash[32040]: cluster 2024-09-13T00:40:14.531746+0000 mon.a (mon.0) 132 : cluster [DBG] osdmap e354: 8 total, 8 up, 8 in 2024-09-13T00:40:15.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:15 smithi088 bash[24895]: cluster 2024-09-13T00:40:14.540209+0000 mon.a (mon.0) 133 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:40:15.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:15 smithi088 bash[24895]: cluster 2024-09-13T00:40:15.530372+0000 mon.a (mon.0) 134 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2024-09-13T00:40:16.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:15 smithi077 bash[80681]: cluster 2024-09-13T00:40:14.540209+0000 mon.a (mon.0) 133 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:40:16.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:15 smithi077 bash[80681]: cluster 2024-09-13T00:40:15.530372+0000 mon.a (mon.0) 134 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2024-09-13T00:40:16.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:15 smithi077 bash[32040]: cluster 2024-09-13T00:40:14.540209+0000 mon.a (mon.0) 133 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:40:16.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:15 smithi077 bash[32040]: cluster 2024-09-13T00:40:15.530372+0000 mon.a (mon.0) 134 : cluster [DBG] osdmap e355: 8 total, 8 up, 8 in 2024-09-13T00:40:16.592 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestGroup.group_create (2063 ms) 2024-09-13T00:40:16.592 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestGroup.group_createPP 2024-09-13T00:40:16.623 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestGroup.group_createPP (31 ms) 2024-09-13T00:40:16.623 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestGroup.add_image 2024-09-13T00:40:16.699 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestGroup.add_image (76 ms) 2024-09-13T00:40:16.699 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestGroup.add_imagePP 2024-09-13T00:40:16.785 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestGroup.add_imagePP (86 ms) 2024-09-13T00:40:16.785 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestGroup.add_snapshot 2024-09-13T00:40:16.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:16 smithi088 bash[24895]: audit 2024-09-13T00:40:15.915103+0000 mon.b (mon.1) 127 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:16.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:16 smithi088 bash[24895]: audit 2024-09-13T00:40:16.283886+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:16.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:16 smithi088 bash[24895]: audit 2024-09-13T00:40:16.297096+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:16.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:16 smithi088 bash[24895]: cluster 2024-09-13T00:40:16.355226+0000 mgr.y (mgr.25304) 56 : cluster [DBG] pgmap v107: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:16.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:16 smithi088 bash[24895]: cluster 2024-09-13T00:40:16.533551+0000 mon.a (mon.0) 137 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2024-09-13T00:40:17.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:16 smithi077 bash[80681]: audit 2024-09-13T00:40:15.915103+0000 mon.b (mon.1) 127 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:17.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:16 smithi077 bash[80681]: audit 2024-09-13T00:40:16.283886+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:17.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:16 smithi077 bash[80681]: audit 2024-09-13T00:40:16.297096+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:17.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:16 smithi077 bash[80681]: cluster 2024-09-13T00:40:16.355226+0000 mgr.y (mgr.25304) 56 : cluster [DBG] pgmap v107: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:17.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:16 smithi077 bash[80681]: cluster 2024-09-13T00:40:16.533551+0000 mon.a (mon.0) 137 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2024-09-13T00:40:17.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:16 smithi077 bash[32040]: audit 2024-09-13T00:40:15.915103+0000 mon.b (mon.1) 127 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:17.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:16 smithi077 bash[32040]: audit 2024-09-13T00:40:16.283886+0000 mon.a (mon.0) 135 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:17.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:16 smithi077 bash[32040]: audit 2024-09-13T00:40:16.297096+0000 mon.a (mon.0) 136 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:17.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:16 smithi077 bash[32040]: cluster 2024-09-13T00:40:16.355226+0000 mgr.y (mgr.25304) 56 : cluster [DBG] pgmap v107: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:17.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:16 smithi077 bash[32040]: cluster 2024-09-13T00:40:16.533551+0000 mon.a (mon.0) 137 : cluster [DBG] osdmap e356: 8 total, 8 up, 8 in 2024-09-13T00:40:17.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:17 smithi088 bash[24895]: cluster 2024-09-13T00:40:17.294752+0000 mon.a (mon.0) 138 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:40:18.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:17 smithi077 bash[80681]: cluster 2024-09-13T00:40:17.294752+0000 mon.a (mon.0) 138 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:40:18.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:17 smithi077 bash[32040]: cluster 2024-09-13T00:40:17.294752+0000 mon.a (mon.0) 138 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:40:18.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:40:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:40:18] "GET /metrics HTTP/1.1" 200 34017 "" "Prometheus/2.43.0" 2024-09-13T00:40:18.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:18 smithi077 bash[80681]: cluster 2024-09-13T00:40:17.587465+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2024-09-13T00:40:18.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:18 smithi077 bash[80681]: cluster 2024-09-13T00:40:18.355906+0000 mgr.y (mgr.25304) 57 : cluster [DBG] pgmap v110: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:18.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:18 smithi077 bash[32040]: cluster 2024-09-13T00:40:17.587465+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2024-09-13T00:40:18.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:18 smithi077 bash[32040]: cluster 2024-09-13T00:40:18.355906+0000 mgr.y (mgr.25304) 57 : cluster [DBG] pgmap v110: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:18.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:18 smithi088 bash[24895]: cluster 2024-09-13T00:40:17.587465+0000 mon.a (mon.0) 139 : cluster [DBG] osdmap e357: 8 total, 8 up, 8 in 2024-09-13T00:40:18.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:18 smithi088 bash[24895]: cluster 2024-09-13T00:40:18.355906+0000 mgr.y (mgr.25304) 57 : cluster [DBG] pgmap v110: 105 pgs: 32 creating+peering, 73 active+clean; 582 KiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:40:18.852 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:18 smithi088 bash[61791]: ts=2024-09-13T00:40:18.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:19.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:19 smithi088 bash[24895]: cluster 2024-09-13T00:40:18.594250+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2024-09-13T00:40:20.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:19 smithi077 bash[80681]: cluster 2024-09-13T00:40:18.594250+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2024-09-13T00:40:20.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:19 smithi077 bash[32040]: cluster 2024-09-13T00:40:18.594250+0000 mon.a (mon.0) 140 : cluster [DBG] osdmap e358: 8 total, 8 up, 8 in 2024-09-13T00:40:21.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:20 smithi077 bash[80681]: cluster 2024-09-13T00:40:19.605407+0000 mon.a (mon.0) 141 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2024-09-13T00:40:21.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:20 smithi077 bash[80681]: cluster 2024-09-13T00:40:20.356605+0000 mgr.y (mgr.25304) 58 : cluster [DBG] pgmap v113: 105 pgs: 105 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 16 KiB/s wr, 188 op/s 2024-09-13T00:40:21.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:20 smithi077 bash[32040]: cluster 2024-09-13T00:40:19.605407+0000 mon.a (mon.0) 141 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2024-09-13T00:40:21.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:20 smithi077 bash[32040]: cluster 2024-09-13T00:40:20.356605+0000 mgr.y (mgr.25304) 58 : cluster [DBG] pgmap v113: 105 pgs: 105 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 16 KiB/s wr, 188 op/s 2024-09-13T00:40:21.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:20 smithi088 bash[24895]: cluster 2024-09-13T00:40:19.605407+0000 mon.a (mon.0) 141 : cluster [DBG] osdmap e359: 8 total, 8 up, 8 in 2024-09-13T00:40:21.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:20 smithi088 bash[24895]: cluster 2024-09-13T00:40:20.356605+0000 mgr.y (mgr.25304) 58 : cluster [DBG] pgmap v113: 105 pgs: 105 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 138 KiB/s rd, 16 KiB/s wr, 188 op/s 2024-09-13T00:40:21.870 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestGroup.add_snapshot (5085 ms) 2024-09-13T00:40:21.870 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestGroup.add_snapshotPP 2024-09-13T00:40:22.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:21 smithi077 bash[32040]: cluster 2024-09-13T00:40:20.615006+0000 mon.a (mon.0) 142 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2024-09-13T00:40:22.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:21 smithi077 bash[32040]: cluster 2024-09-13T00:40:20.846270+0000 mon.a (mon.0) 143 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2024-09-13T00:40:22.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:21 smithi077 bash[80681]: cluster 2024-09-13T00:40:20.615006+0000 mon.a (mon.0) 142 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2024-09-13T00:40:22.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:21 smithi077 bash[80681]: cluster 2024-09-13T00:40:20.846270+0000 mon.a (mon.0) 143 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2024-09-13T00:40:22.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:21 smithi088 bash[24895]: cluster 2024-09-13T00:40:20.615006+0000 mon.a (mon.0) 142 : cluster [DBG] osdmap e360: 8 total, 8 up, 8 in 2024-09-13T00:40:22.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:21 smithi088 bash[24895]: cluster 2024-09-13T00:40:20.846270+0000 mon.a (mon.0) 143 : cluster [DBG] osdmap e361: 8 total, 8 up, 8 in 2024-09-13T00:40:22.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:22 smithi088 bash[61791]: ts=2024-09-13T00:40:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:23.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[80681]: audit 2024-09-13T00:40:21.786555+0000 mon.a (mon.0) 144 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:23.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[80681]: audit 2024-09-13T00:40:21.793436+0000 mon.a (mon.0) 145 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:23.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[80681]: audit 2024-09-13T00:40:21.834741+0000 mon.a (mon.0) 146 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:23.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[80681]: audit 2024-09-13T00:40:21.841506+0000 mon.a (mon.0) 147 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:23.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[80681]: cluster 2024-09-13T00:40:21.845901+0000 mon.a (mon.0) 148 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2024-09-13T00:40:23.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[80681]: audit 2024-09-13T00:40:22.293244+0000 mon.b (mon.1) 128 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:40:23.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[80681]: audit 2024-09-13T00:40:22.295173+0000 mon.b (mon.1) 129 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:40:23.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[80681]: audit 2024-09-13T00:40:22.304395+0000 mon.a (mon.0) 149 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:23.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[80681]: cluster 2024-09-13T00:40:22.357074+0000 mgr.y (mgr.25304) 59 : cluster [DBG] pgmap v117: 105 pgs: 105 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 147 KiB/s rd, 17 KiB/s wr, 199 op/s 2024-09-13T00:40:23.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[32040]: audit 2024-09-13T00:40:21.786555+0000 mon.a (mon.0) 144 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:23.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[32040]: audit 2024-09-13T00:40:21.793436+0000 mon.a (mon.0) 145 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:23.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[32040]: audit 2024-09-13T00:40:21.834741+0000 mon.a (mon.0) 146 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:23.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[32040]: audit 2024-09-13T00:40:21.841506+0000 mon.a (mon.0) 147 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:23.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[32040]: cluster 2024-09-13T00:40:21.845901+0000 mon.a (mon.0) 148 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2024-09-13T00:40:23.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[32040]: audit 2024-09-13T00:40:22.293244+0000 mon.b (mon.1) 128 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:40:23.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[32040]: audit 2024-09-13T00:40:22.295173+0000 mon.b (mon.1) 129 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:40:23.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[32040]: audit 2024-09-13T00:40:22.304395+0000 mon.a (mon.0) 149 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:23.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:22 smithi077 bash[32040]: cluster 2024-09-13T00:40:22.357074+0000 mgr.y (mgr.25304) 59 : cluster [DBG] pgmap v117: 105 pgs: 105 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 147 KiB/s rd, 17 KiB/s wr, 199 op/s 2024-09-13T00:40:23.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:22 smithi088 bash[24895]: audit 2024-09-13T00:40:21.786555+0000 mon.a (mon.0) 144 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:23.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:22 smithi088 bash[24895]: audit 2024-09-13T00:40:21.793436+0000 mon.a (mon.0) 145 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:23.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:22 smithi088 bash[24895]: audit 2024-09-13T00:40:21.834741+0000 mon.a (mon.0) 146 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:23.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:22 smithi088 bash[24895]: audit 2024-09-13T00:40:21.841506+0000 mon.a (mon.0) 147 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:23.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:22 smithi088 bash[24895]: cluster 2024-09-13T00:40:21.845901+0000 mon.a (mon.0) 148 : cluster [DBG] osdmap e362: 8 total, 8 up, 8 in 2024-09-13T00:40:23.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:22 smithi088 bash[24895]: audit 2024-09-13T00:40:22.293244+0000 mon.b (mon.1) 128 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:40:23.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:22 smithi088 bash[24895]: audit 2024-09-13T00:40:22.295173+0000 mon.b (mon.1) 129 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:40:23.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:22 smithi088 bash[24895]: audit 2024-09-13T00:40:22.304395+0000 mon.a (mon.0) 149 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:23.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:22 smithi088 bash[24895]: cluster 2024-09-13T00:40:22.357074+0000 mgr.y (mgr.25304) 59 : cluster [DBG] pgmap v117: 105 pgs: 105 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 147 KiB/s rd, 17 KiB/s wr, 199 op/s 2024-09-13T00:40:23.754 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph orch upgrade resume' 2024-09-13T00:40:23.846 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:23 smithi088 bash[61791]: ts=2024-09-13T00:40:23.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:24.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:23 smithi077 bash[80681]: cluster 2024-09-13T00:40:22.854040+0000 mon.a (mon.0) 150 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2024-09-13T00:40:24.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:23 smithi077 bash[32040]: cluster 2024-09-13T00:40:22.854040+0000 mon.a (mon.0) 150 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2024-09-13T00:40:24.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:23 smithi088 bash[24895]: cluster 2024-09-13T00:40:22.854040+0000 mon.a (mon.0) 150 : cluster [DBG] osdmap e363: 8 total, 8 up, 8 in 2024-09-13T00:40:24.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:24 smithi088 bash[61791]: ts=2024-09-13T00:40:24.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:24.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:24 smithi088 bash[61791]: ts=2024-09-13T00:40:24.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:25.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:24 smithi077 bash[80681]: cluster 2024-09-13T00:40:23.863456+0000 mon.a (mon.0) 151 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2024-09-13T00:40:25.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:24 smithi077 bash[80681]: cluster 2024-09-13T00:40:24.358315+0000 mgr.y (mgr.25304) 60 : cluster [DBG] pgmap v120: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 13 KiB/s wr, 171 op/s 2024-09-13T00:40:25.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:24 smithi077 bash[32040]: cluster 2024-09-13T00:40:23.863456+0000 mon.a (mon.0) 151 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2024-09-13T00:40:25.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:24 smithi077 bash[32040]: cluster 2024-09-13T00:40:24.358315+0000 mgr.y (mgr.25304) 60 : cluster [DBG] pgmap v120: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 13 KiB/s wr, 171 op/s 2024-09-13T00:40:25.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:24 smithi088 bash[24895]: cluster 2024-09-13T00:40:23.863456+0000 mon.a (mon.0) 151 : cluster [DBG] osdmap e364: 8 total, 8 up, 8 in 2024-09-13T00:40:25.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:24 smithi088 bash[24895]: cluster 2024-09-13T00:40:24.358315+0000 mgr.y (mgr.25304) 60 : cluster [DBG] pgmap v120: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 127 KiB/s rd, 13 KiB/s wr, 171 op/s 2024-09-13T00:40:26.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:25 smithi077 bash[80681]: cluster 2024-09-13T00:40:24.875146+0000 mon.a (mon.0) 152 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2024-09-13T00:40:26.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:25 smithi077 bash[80681]: cluster 2024-09-13T00:40:25.849324+0000 mon.a (mon.0) 153 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2024-09-13T00:40:26.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:25 smithi077 bash[32040]: cluster 2024-09-13T00:40:24.875146+0000 mon.a (mon.0) 152 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2024-09-13T00:40:26.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:25 smithi077 bash[32040]: cluster 2024-09-13T00:40:25.849324+0000 mon.a (mon.0) 153 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2024-09-13T00:40:26.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:25 smithi088 bash[24895]: cluster 2024-09-13T00:40:24.875146+0000 mon.a (mon.0) 152 : cluster [DBG] osdmap e365: 8 total, 8 up, 8 in 2024-09-13T00:40:26.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:25 smithi088 bash[24895]: cluster 2024-09-13T00:40:25.849324+0000 mon.a (mon.0) 153 : cluster [DBG] osdmap e366: 8 total, 8 up, 8 in 2024-09-13T00:40:27.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:26 smithi077 bash[80681]: cluster 2024-09-13T00:40:26.359016+0000 mgr.y (mgr.25304) 61 : cluster [DBG] pgmap v123: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 12 KiB/s wr, 160 op/s 2024-09-13T00:40:27.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:26 smithi077 bash[32040]: cluster 2024-09-13T00:40:26.359016+0000 mgr.y (mgr.25304) 61 : cluster [DBG] pgmap v123: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 12 KiB/s wr, 160 op/s 2024-09-13T00:40:27.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:26 smithi088 bash[24895]: cluster 2024-09-13T00:40:26.359016+0000 mgr.y (mgr.25304) 61 : cluster [DBG] pgmap v123: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 12 KiB/s wr, 160 op/s 2024-09-13T00:40:27.947 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestGroup.add_snapshotPP (6077 ms) 2024-09-13T00:40:28.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:27 smithi077 bash[80681]: cluster 2024-09-13T00:40:26.887825+0000 mon.a (mon.0) 154 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2024-09-13T00:40:28.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:27 smithi077 bash[32040]: cluster 2024-09-13T00:40:26.887825+0000 mon.a (mon.0) 154 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2024-09-13T00:40:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:27 smithi088 bash[24895]: cluster 2024-09-13T00:40:26.887825+0000 mon.a (mon.0) 154 : cluster [DBG] osdmap e367: 8 total, 8 up, 8 in 2024-09-13T00:40:28.553 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:40:28.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:40:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:40:28] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.43.0" 2024-09-13T00:40:28.911 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 6 tests from TestGroup (13418 ms total) 2024-09-13T00:40:28.911 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:40:28.912 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 31 tests from TestDeepCopy 2024-09-13T00:40:29.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:28 smithi088 bash[24895]: cluster 2024-09-13T00:40:27.895403+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e368: 8 total, 8 up, 8 in 2024-09-13T00:40:29.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:28 smithi088 bash[24895]: cluster 2024-09-13T00:40:28.359672+0000 mgr.y (mgr.25304) 62 : cluster [DBG] pgmap v126: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:40:29.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:28 smithi088 bash[24895]: audit 2024-09-13T00:40:28.409114+0000 mon.b (mon.1) 130 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:29.102 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:28 smithi088 bash[61791]: ts=2024-09-13T00:40:28.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:29.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:28 smithi077 bash[80681]: cluster 2024-09-13T00:40:27.895403+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e368: 8 total, 8 up, 8 in 2024-09-13T00:40:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:28 smithi077 bash[80681]: cluster 2024-09-13T00:40:28.359672+0000 mgr.y (mgr.25304) 62 : cluster [DBG] pgmap v126: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:40:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:28 smithi077 bash[80681]: audit 2024-09-13T00:40:28.409114+0000 mon.b (mon.1) 130 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:29.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:28 smithi077 bash[32040]: cluster 2024-09-13T00:40:27.895403+0000 mon.a (mon.0) 155 : cluster [DBG] osdmap e368: 8 total, 8 up, 8 in 2024-09-13T00:40:29.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:28 smithi077 bash[32040]: cluster 2024-09-13T00:40:28.359672+0000 mgr.y (mgr.25304) 62 : cluster [DBG] pgmap v126: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:40:29.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:28 smithi077 bash[32040]: audit 2024-09-13T00:40:28.409114+0000 mon.b (mon.1) 130 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:30.223 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:29 smithi077 bash[80681]: cluster 2024-09-13T00:40:28.906727+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2024-09-13T00:40:30.223 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:29 smithi077 bash[32040]: cluster 2024-09-13T00:40:28.906727+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2024-09-13T00:40:30.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:29 smithi088 bash[24895]: cluster 2024-09-13T00:40:28.906727+0000 mon.a (mon.0) 156 : cluster [DBG] osdmap e369: 8 total, 8 up, 8 in 2024-09-13T00:40:30.942 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.Empty 2024-09-13T00:40:31.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:30 smithi077 bash[80681]: cluster 2024-09-13T00:40:29.917578+0000 mon.a (mon.0) 157 : cluster [DBG] mgrmap e39: y(active, since 91s), standbys: x 2024-09-13T00:40:31.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:30 smithi077 bash[80681]: cluster 2024-09-13T00:40:29.929424+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2024-09-13T00:40:31.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:30 smithi077 bash[80681]: audit 2024-09-13T00:40:29.941612+0000 mon.c (mon.2) 237 : audit [INF] from='client.? 172.21.15.77:0/3820756590' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:31.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:30 smithi077 bash[80681]: audit 2024-09-13T00:40:29.942974+0000 mon.a (mon.0) 159 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:31.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:30 smithi077 bash[80681]: cluster 2024-09-13T00:40:30.360417+0000 mgr.y (mgr.25304) 63 : cluster [DBG] pgmap v129: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:40:31.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:30 smithi077 bash[32040]: cluster 2024-09-13T00:40:29.917578+0000 mon.a (mon.0) 157 : cluster [DBG] mgrmap e39: y(active, since 91s), standbys: x 2024-09-13T00:40:31.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:30 smithi077 bash[32040]: cluster 2024-09-13T00:40:29.929424+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2024-09-13T00:40:31.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:30 smithi077 bash[32040]: audit 2024-09-13T00:40:29.941612+0000 mon.c (mon.2) 237 : audit [INF] from='client.? 172.21.15.77:0/3820756590' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:31.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:30 smithi077 bash[32040]: audit 2024-09-13T00:40:29.942974+0000 mon.a (mon.0) 159 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:31.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:30 smithi077 bash[32040]: cluster 2024-09-13T00:40:30.360417+0000 mgr.y (mgr.25304) 63 : cluster [DBG] pgmap v129: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:40:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:30 smithi088 bash[24895]: cluster 2024-09-13T00:40:29.917578+0000 mon.a (mon.0) 157 : cluster [DBG] mgrmap e39: y(active, since 91s), standbys: x 2024-09-13T00:40:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:30 smithi088 bash[24895]: cluster 2024-09-13T00:40:29.929424+0000 mon.a (mon.0) 158 : cluster [DBG] osdmap e370: 8 total, 8 up, 8 in 2024-09-13T00:40:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:30 smithi088 bash[24895]: audit 2024-09-13T00:40:29.941612+0000 mon.c (mon.2) 237 : audit [INF] from='client.? 172.21.15.77:0/3820756590' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:31.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:30 smithi088 bash[24895]: audit 2024-09-13T00:40:29.942974+0000 mon.a (mon.0) 159 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:40:31.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:30 smithi088 bash[24895]: cluster 2024-09-13T00:40:30.360417+0000 mgr.y (mgr.25304) 63 : cluster [DBG] pgmap v129: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:40:32.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:31 smithi077 bash[80681]: cluster 2024-09-13T00:40:30.923763+0000 mon.a (mon.0) 160 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:40:32.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:31 smithi077 bash[80681]: audit 2024-09-13T00:40:30.942722+0000 mon.a (mon.0) 161 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:40:32.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:31 smithi077 bash[80681]: cluster 2024-09-13T00:40:30.942974+0000 mon.a (mon.0) 162 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2024-09-13T00:40:32.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:31 smithi077 bash[32040]: cluster 2024-09-13T00:40:30.923763+0000 mon.a (mon.0) 160 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:40:32.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:31 smithi077 bash[32040]: audit 2024-09-13T00:40:30.942722+0000 mon.a (mon.0) 161 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:40:32.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:31 smithi077 bash[32040]: cluster 2024-09-13T00:40:30.942974+0000 mon.a (mon.0) 162 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2024-09-13T00:40:32.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:31 smithi088 bash[24895]: cluster 2024-09-13T00:40:30.923763+0000 mon.a (mon.0) 160 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:40:32.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:31 smithi088 bash[24895]: audit 2024-09-13T00:40:30.942722+0000 mon.a (mon.0) 161 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:40:32.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:31 smithi088 bash[24895]: cluster 2024-09-13T00:40:30.942974+0000 mon.a (mon.0) 162 : cluster [DBG] osdmap e371: 8 total, 8 up, 8 in 2024-09-13T00:40:32.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:32 smithi088 bash[61791]: ts=2024-09-13T00:40:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:32.677 INFO:teuthology.orchestra.run.smithi077.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:40:33.145 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.Empty (2203 ms) 2024-09-13T00:40:33.146 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.NoSnaps 2024-09-13T00:40:33.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[80681]: cluster 2024-09-13T00:40:31.947306+0000 mon.a (mon.0) 163 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2024-09-13T00:40:33.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[80681]: cluster 2024-09-13T00:40:32.360857+0000 mgr.y (mgr.25304) 64 : cluster [DBG] pgmap v132: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:40:33.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[80681]: audit 2024-09-13T00:40:32.677334+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:33.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[80681]: audit 2024-09-13T00:40:32.680800+0000 mon.b (mon.1) 131 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:33.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[80681]: audit 2024-09-13T00:40:32.684297+0000 mon.b (mon.1) 132 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:40:33.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[80681]: audit 2024-09-13T00:40:32.685930+0000 mon.b (mon.1) 133 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:40:33.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[80681]: audit 2024-09-13T00:40:32.693601+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:33.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[80681]: audit 2024-09-13T00:40:32.777332+0000 mon.b (mon.1) 134 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:33.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[80681]: audit 2024-09-13T00:40:32.782324+0000 mon.b (mon.1) 135 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:40:33.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[80681]: audit 2024-09-13T00:40:32.784671+0000 mon.b (mon.1) 136 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T00:40:33.282 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[80681]: audit 2024-09-13T00:40:32.785892+0000 mon.b (mon.1) 137 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-13T00:40:33.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[32040]: cluster 2024-09-13T00:40:31.947306+0000 mon.a (mon.0) 163 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2024-09-13T00:40:33.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[32040]: cluster 2024-09-13T00:40:32.360857+0000 mgr.y (mgr.25304) 64 : cluster [DBG] pgmap v132: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:40:33.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[32040]: audit 2024-09-13T00:40:32.677334+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:33.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[32040]: audit 2024-09-13T00:40:32.680800+0000 mon.b (mon.1) 131 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:33.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[32040]: audit 2024-09-13T00:40:32.684297+0000 mon.b (mon.1) 132 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:40:33.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[32040]: audit 2024-09-13T00:40:32.685930+0000 mon.b (mon.1) 133 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:40:33.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[32040]: audit 2024-09-13T00:40:32.693601+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:33.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[32040]: audit 2024-09-13T00:40:32.777332+0000 mon.b (mon.1) 134 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:33.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[32040]: audit 2024-09-13T00:40:32.782324+0000 mon.b (mon.1) 135 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:40:33.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[32040]: audit 2024-09-13T00:40:32.784671+0000 mon.b (mon.1) 136 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T00:40:33.283 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:32 smithi077 bash[32040]: audit 2024-09-13T00:40:32.785892+0000 mon.b (mon.1) 137 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-13T00:40:33.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:32 smithi088 bash[24895]: cluster 2024-09-13T00:40:31.947306+0000 mon.a (mon.0) 163 : cluster [DBG] osdmap e372: 8 total, 8 up, 8 in 2024-09-13T00:40:33.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:32 smithi088 bash[24895]: cluster 2024-09-13T00:40:32.360857+0000 mgr.y (mgr.25304) 64 : cluster [DBG] pgmap v132: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:40:33.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:32 smithi088 bash[24895]: audit 2024-09-13T00:40:32.677334+0000 mon.a (mon.0) 164 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:33.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:32 smithi088 bash[24895]: audit 2024-09-13T00:40:32.680800+0000 mon.b (mon.1) 131 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:33.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:32 smithi088 bash[24895]: audit 2024-09-13T00:40:32.684297+0000 mon.b (mon.1) 132 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:40:33.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:32 smithi088 bash[24895]: audit 2024-09-13T00:40:32.685930+0000 mon.b (mon.1) 133 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:40:33.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:32 smithi088 bash[24895]: audit 2024-09-13T00:40:32.693601+0000 mon.a (mon.0) 165 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:33.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:32 smithi088 bash[24895]: audit 2024-09-13T00:40:32.777332+0000 mon.b (mon.1) 134 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:33.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:32 smithi088 bash[24895]: audit 2024-09-13T00:40:32.782324+0000 mon.b (mon.1) 135 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:40:33.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:32 smithi088 bash[24895]: audit 2024-09-13T00:40:32.784671+0000 mon.b (mon.1) 136 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T00:40:33.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:32 smithi088 bash[24895]: audit 2024-09-13T00:40:32.785892+0000 mon.b (mon.1) 137 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-13T00:40:33.538 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.NoSnaps (393 ms) 2024-09-13T00:40:33.538 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.Snaps 2024-09-13T00:40:33.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:33 smithi088 bash[61791]: ts=2024-09-13T00:40:33.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:33.925 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'echo wait for majority of mons to upgrade' 2024-09-13T00:40:34.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:33 smithi077 bash[80681]: audit 2024-09-13T00:40:32.671151+0000 mgr.y (mgr.25304) 65 : audit [DBG] from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:40:34.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:33 smithi077 bash[80681]: cephadm 2024-09-13T00:40:32.671796+0000 mgr.y (mgr.25304) 66 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:40:34.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:33 smithi077 bash[80681]: cephadm 2024-09-13T00:40:32.787283+0000 mgr.y (mgr.25304) 67 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-09-13T00:40:34.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:33 smithi077 bash[80681]: cluster 2024-09-13T00:40:32.942202+0000 mon.a (mon.0) 166 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:40:34.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:33 smithi077 bash[80681]: cluster 2024-09-13T00:40:32.956048+0000 mon.a (mon.0) 167 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2024-09-13T00:40:34.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:33 smithi077 bash[32040]: audit 2024-09-13T00:40:32.671151+0000 mgr.y (mgr.25304) 65 : audit [DBG] from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:40:34.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:33 smithi077 bash[32040]: cephadm 2024-09-13T00:40:32.671796+0000 mgr.y (mgr.25304) 66 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:40:34.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:33 smithi077 bash[32040]: cephadm 2024-09-13T00:40:32.787283+0000 mgr.y (mgr.25304) 67 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-09-13T00:40:34.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:33 smithi077 bash[32040]: cluster 2024-09-13T00:40:32.942202+0000 mon.a (mon.0) 166 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:40:34.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:33 smithi077 bash[32040]: cluster 2024-09-13T00:40:32.956048+0000 mon.a (mon.0) 167 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2024-09-13T00:40:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:33 smithi088 bash[24895]: audit 2024-09-13T00:40:32.671151+0000 mgr.y (mgr.25304) 65 : audit [DBG] from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:40:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:33 smithi088 bash[24895]: cephadm 2024-09-13T00:40:32.671796+0000 mgr.y (mgr.25304) 66 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:40:34.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:33 smithi088 bash[24895]: cephadm 2024-09-13T00:40:32.787283+0000 mgr.y (mgr.25304) 67 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-09-13T00:40:34.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:33 smithi088 bash[24895]: cluster 2024-09-13T00:40:32.942202+0000 mon.a (mon.0) 166 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:40:34.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:33 smithi088 bash[24895]: cluster 2024-09-13T00:40:32.956048+0000 mon.a (mon.0) 167 : cluster [DBG] osdmap e373: 8 total, 8 up, 8 in 2024-09-13T00:40:34.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:34 smithi088 bash[61791]: ts=2024-09-13T00:40:34.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:34.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:34 smithi088 bash[61791]: ts=2024-09-13T00:40:34.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:35.260 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:34 smithi077 bash[80681]: cluster 2024-09-13T00:40:33.966157+0000 mon.a (mon.0) 168 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2024-09-13T00:40:35.260 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:34 smithi077 bash[80681]: cluster 2024-09-13T00:40:34.362000+0000 mgr.y (mgr.25304) 68 : cluster [DBG] pgmap v135: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-13T00:40:35.260 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:34 smithi077 bash[32040]: cluster 2024-09-13T00:40:33.966157+0000 mon.a (mon.0) 168 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2024-09-13T00:40:35.260 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:34 smithi077 bash[32040]: cluster 2024-09-13T00:40:34.362000+0000 mgr.y (mgr.25304) 68 : cluster [DBG] pgmap v135: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-13T00:40:35.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:34 smithi088 bash[24895]: cluster 2024-09-13T00:40:33.966157+0000 mon.a (mon.0) 168 : cluster [DBG] osdmap e374: 8 total, 8 up, 8 in 2024-09-13T00:40:35.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:34 smithi088 bash[24895]: cluster 2024-09-13T00:40:34.362000+0000 mgr.y (mgr.25304) 68 : cluster [DBG] pgmap v135: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-13T00:40:36.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:35 smithi077 bash[80681]: cluster 2024-09-13T00:40:34.972038+0000 mon.a (mon.0) 169 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2024-09-13T00:40:36.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:35 smithi077 bash[80681]: cluster 2024-09-13T00:40:35.853994+0000 mon.a (mon.0) 170 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2024-09-13T00:40:36.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:35 smithi077 bash[80681]: audit 2024-09-13T00:40:35.921797+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:36.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:35 smithi077 bash[80681]: audit 2024-09-13T00:40:35.926266+0000 mon.b (mon.1) 138 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:40:36.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:35 smithi077 bash[80681]: audit 2024-09-13T00:40:35.928180+0000 mon.b (mon.1) 139 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:40:36.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:35 smithi077 bash[80681]: audit 2024-09-13T00:40:35.929587+0000 mon.b (mon.1) 140 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:40:36.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:35 smithi077 bash[32040]: cluster 2024-09-13T00:40:34.972038+0000 mon.a (mon.0) 169 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2024-09-13T00:40:36.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:35 smithi077 bash[32040]: cluster 2024-09-13T00:40:35.853994+0000 mon.a (mon.0) 170 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2024-09-13T00:40:36.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:35 smithi077 bash[32040]: audit 2024-09-13T00:40:35.921797+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:36.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:35 smithi077 bash[32040]: audit 2024-09-13T00:40:35.926266+0000 mon.b (mon.1) 138 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:40:36.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:35 smithi077 bash[32040]: audit 2024-09-13T00:40:35.928180+0000 mon.b (mon.1) 139 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:40:36.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:35 smithi077 bash[32040]: audit 2024-09-13T00:40:35.929587+0000 mon.b (mon.1) 140 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:40:36.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:35 smithi088 bash[24895]: cluster 2024-09-13T00:40:34.972038+0000 mon.a (mon.0) 169 : cluster [DBG] osdmap e375: 8 total, 8 up, 8 in 2024-09-13T00:40:36.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:35 smithi088 bash[24895]: cluster 2024-09-13T00:40:35.853994+0000 mon.a (mon.0) 170 : cluster [DBG] osdmap e376: 8 total, 8 up, 8 in 2024-09-13T00:40:36.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:35 smithi088 bash[24895]: audit 2024-09-13T00:40:35.921797+0000 mon.a (mon.0) 171 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:36.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:35 smithi088 bash[24895]: audit 2024-09-13T00:40:35.926266+0000 mon.b (mon.1) 138 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:40:36.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:35 smithi088 bash[24895]: audit 2024-09-13T00:40:35.928180+0000 mon.b (mon.1) 139 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:40:36.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:35 smithi088 bash[24895]: audit 2024-09-13T00:40:35.929587+0000 mon.b (mon.1) 140 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:40:37.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:36 smithi077 bash[80681]: cephadm 2024-09-13T00:40:35.914585+0000 mgr.y (mgr.25304) 69 : cephadm [INF] Upgrade: Updating mon.c 2024-09-13T00:40:37.584 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:36 smithi077 bash[80681]: cephadm 2024-09-13T00:40:35.931869+0000 mgr.y (mgr.25304) 70 : cephadm [INF] Deploying daemon mon.c on smithi077 2024-09-13T00:40:37.584 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:36 smithi077 bash[80681]: cluster 2024-09-13T00:40:36.362737+0000 mgr.y (mgr.25304) 71 : cluster [DBG] pgmap v138: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-13T00:40:37.584 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:36 smithi077 bash[80681]: cluster 2024-09-13T00:40:36.857635+0000 mon.a (mon.0) 172 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2024-09-13T00:40:37.585 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:36 smithi077 bash[32040]: cephadm 2024-09-13T00:40:35.914585+0000 mgr.y (mgr.25304) 69 : cephadm [INF] Upgrade: Updating mon.c 2024-09-13T00:40:37.585 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:36 smithi077 bash[32040]: cephadm 2024-09-13T00:40:35.931869+0000 mgr.y (mgr.25304) 70 : cephadm [INF] Deploying daemon mon.c on smithi077 2024-09-13T00:40:37.585 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:36 smithi077 bash[32040]: cluster 2024-09-13T00:40:36.362737+0000 mgr.y (mgr.25304) 71 : cluster [DBG] pgmap v138: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-13T00:40:37.585 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:36 smithi077 bash[32040]: cluster 2024-09-13T00:40:36.857635+0000 mon.a (mon.0) 172 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2024-09-13T00:40:37.585 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.Snaps (3800 ms) 2024-09-13T00:40:37.585 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.SnapDiscard 2024-09-13T00:40:37.586 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:36 smithi088 bash[24895]: cephadm 2024-09-13T00:40:35.914585+0000 mgr.y (mgr.25304) 69 : cephadm [INF] Upgrade: Updating mon.c 2024-09-13T00:40:37.586 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:36 smithi088 bash[24895]: cephadm 2024-09-13T00:40:35.931869+0000 mgr.y (mgr.25304) 70 : cephadm [INF] Deploying daemon mon.c on smithi077 2024-09-13T00:40:37.586 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:36 smithi088 bash[24895]: cluster 2024-09-13T00:40:36.362737+0000 mgr.y (mgr.25304) 71 : cluster [DBG] pgmap v138: 105 pgs: 105 active+clean; 49 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-13T00:40:37.586 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:36 smithi088 bash[24895]: cluster 2024-09-13T00:40:36.857635+0000 mon.a (mon.0) 172 : cluster [DBG] osdmap e377: 8 total, 8 up, 8 in 2024-09-13T00:40:38.676 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:38 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:38.676 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:40:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:40:38] "GET /metrics HTTP/1.1" 200 36425 "" "Prometheus/2.43.0" 2024-09-13T00:40:38.676 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:40:38 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:38.677 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:40:38 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:38.677 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:40:38 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:38.677 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:40:38 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:38.678 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:40:38 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:38.678 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:38 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:38.678 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:38 smithi077 systemd[1]: Stopping Ceph mon.c for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:40:38.679 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:40:38 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:38.679 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:40:38 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:39.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:38 smithi077 bash[32040]: debug 2024-09-13T00:40:38.803+0000 7f3d82360640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T00:40:39.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:38 smithi077 bash[32040]: debug 2024-09-13T00:40:38.803+0000 7f3d82360640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-09-13T00:40:39.101 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:38 smithi088 bash[61791]: ts=2024-09-13T00:40:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:39.897 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:39 smithi077 bash[83012]: ceph-9218306c-7165-11ef-bceb-c7b262605968-mon-c 2024-09-13T00:40:39.897 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:39 smithi077 bash[83073]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-mon-c 2024-09-13T00:40:39.897 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:39 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-13T00:40:39.897 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:39 smithi077 systemd[1]: Stopped Ceph mon.c for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:40:40.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:39 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:40.278 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:40:39 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:40.279 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:40:39 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:40.279 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:40:39 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:40.279 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:40:39 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:40.280 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:40:39 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:40.280 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:40:39 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:40.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:39 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:40.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:40 smithi077 systemd[1]: Started Ceph mon.c for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:40:40.281 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:40:39 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:40:40.285 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:40:42.487 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.295+0000 7f998130eb00 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-13T00:40:42.487 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.295+0000 7f998130eb00 0 ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable), process ceph-mon, pid 7 2024-09-13T00:40:42.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.295+0000 7f998130eb00 0 pidfile_write: ignore empty --pid-file 2024-09-13T00:40:42.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 0 load: jerasure load: lrc 2024-09-13T00:40:42.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: RocksDB version: 7.9.2 2024-09-13T00:40:42.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Git sha 0 2024-09-13T00:40:42.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Compile date 2024-09-10 15:58:06 2024-09-13T00:40:42.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: DB SUMMARY 2024-09-13T00:40:42.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: DB Session ID: 455VXH5NT5NKZ538NZ7S 2024-09-13T00:40:42.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: CURRENT file: CURRENT 2024-09-13T00:40:42.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: IDENTITY file: IDENTITY 2024-09-13T00:40:42.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 1861 Bytes 2024-09-13T00:40:42.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000039.sst 2024-09-13T00:40:42.488 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000037.log size: 5774490 ; 2024-09-13T00:40:42.489 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.error_if_exists: 0 2024-09-13T00:40:42.489 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.create_if_missing: 0 2024-09-13T00:40:42.489 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.paranoid_checks: 1 2024-09-13T00:40:42.489 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T00:40:42.489 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T00:40:42.489 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T00:40:42.489 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.env: 0x5625660b5bc0 2024-09-13T00:40:42.489 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.fs: PosixFileSystem 2024-09-13T00:40:42.489 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.info_log: 0x562566c94be0 2024-09-13T00:40:42.489 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-13T00:40:42.489 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.statistics: (nil) 2024-09-13T00:40:42.489 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.use_fsync: 0 2024-09-13T00:40:42.489 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_log_file_size: 0 2024-09-13T00:40:42.490 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T00:40:42.490 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T00:40:42.490 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-13T00:40:42.490 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-13T00:40:42.490 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.allow_fallocate: 1 2024-09-13T00:40:42.490 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-13T00:40:42.490 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-13T00:40:42.490 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.use_direct_reads: 0 2024-09-13T00:40:42.490 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T00:40:42.490 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.create_missing_column_families: 0 2024-09-13T00:40:42.490 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.db_log_dir: 2024-09-13T00:40:42.490 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.wal_dir: 2024-09-13T00:40:42.490 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T00:40:42.491 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T00:40:42.491 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T00:40:42.491 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T00:40:42.491 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T00:40:42.491 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T00:40:42.491 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.advise_random_on_open: 1 2024-09-13T00:40:42.491 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-13T00:40:42.491 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.write_buffer_manager: 0x562566ca2320 2024-09-13T00:40:42.491 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T00:40:42.491 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T00:40:42.491 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T00:40:42.491 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.rate_limiter: (nil) 2024-09-13T00:40:42.491 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T00:40:42.492 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-13T00:40:42.496 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-13T00:40:42.497 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-13T00:40:42.497 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.unordered_write: 0 2024-09-13T00:40:42.497 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T00:40:42.497 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T00:40:42.497 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T00:40:42.497 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T00:40:42.497 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.row_cache: None 2024-09-13T00:40:42.497 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.wal_filter: None 2024-09-13T00:40:42.497 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T00:40:42.497 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-13T00:40:42.497 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.two_write_queues: 0 2024-09-13T00:40:42.497 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.manual_wal_flush: 0 2024-09-13T00:40:42.498 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.wal_compression: 0 2024-09-13T00:40:42.498 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.atomic_flush: 0 2024-09-13T00:40:42.498 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T00:40:42.498 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T00:40:42.498 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T00:40:42.498 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.log_readahead_size: 0 2024-09-13T00:40:42.498 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T00:40:42.498 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-13T00:40:42.498 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T00:40:42.498 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T00:40:42.498 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-13T00:40:42.498 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-13T00:40:42.498 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T00:40:42.499 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_background_jobs: 2 2024-09-13T00:40:42.499 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_background_compactions: -1 2024-09-13T00:40:42.499 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_subcompactions: 1 2024-09-13T00:40:42.499 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T00:40:42.499 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T00:40:42.499 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T00:40:42.499 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_total_wal_size: 0 2024-09-13T00:40:42.499 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T00:40:42.499 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T00:40:42.500 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T00:40:42.500 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T00:40:42.500 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_open_files: -1 2024-09-13T00:40:42.500 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.bytes_per_sync: 0 2024-09-13T00:40:42.500 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T00:40:42.500 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T00:40:42.500 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-13T00:40:42.500 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_background_flushes: -1 2024-09-13T00:40:42.500 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Compression algorithms supported: 2024-09-13T00:40:42.500 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: kZSTD supported: 0 2024-09-13T00:40:42.500 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: kXpressCompression supported: 0 2024-09-13T00:40:42.500 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: kBZip2Compression supported: 0 2024-09-13T00:40:42.501 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T00:40:42.501 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: kLZ4Compression supported: 1 2024-09-13T00:40:42.501 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: kZlibCompression supported: 1 2024-09-13T00:40:42.501 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-13T00:40:42.501 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: kSnappyCompression supported: 1 2024-09-13T00:40:42.501 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T00:40:42.501 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T00:40:42.501 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-09-13T00:40:42.501 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T00:40:42.501 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T00:40:42.501 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.merge_operator: 2024-09-13T00:40:42.501 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compaction_filter: None 2024-09-13T00:40:42.502 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compaction_filter_factory: None 2024-09-13T00:40:42.502 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-13T00:40:42.502 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T00:40:42.502 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T00:40:42.502 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562566c947e0) 2024-09-13T00:40:42.502 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: cache_index_and_filter_blocks: 1 2024-09-13T00:40:42.502 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T00:40:42.502 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T00:40:42.502 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: pin_top_level_index_and_filter: 1 2024-09-13T00:40:42.502 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: index_type: 0 2024-09-13T00:40:42.502 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: data_block_index_type: 0 2024-09-13T00:40:42.502 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: index_shortening: 1 2024-09-13T00:40:42.502 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: data_block_hash_table_util_ratio: 0.750000 2024-09-13T00:40:42.503 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: checksum: 4 2024-09-13T00:40:42.503 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: no_block_cache: 0 2024-09-13T00:40:42.503 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: block_cache: 0x562566c8b350 2024-09-13T00:40:42.503 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: block_cache_name: BinnedLRUCache 2024-09-13T00:40:42.503 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: block_cache_options: 2024-09-13T00:40:42.503 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: capacity : 536870912 2024-09-13T00:40:42.503 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: num_shard_bits : 4 2024-09-13T00:40:42.503 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: strict_capacity_limit : 0 2024-09-13T00:40:42.503 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: high_pri_pool_ratio: 0.000 2024-09-13T00:40:42.503 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: block_cache_compressed: (nil) 2024-09-13T00:40:42.503 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: persistent_cache: (nil) 2024-09-13T00:40:42.503 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: block_size: 4096 2024-09-13T00:40:42.503 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: block_size_deviation: 10 2024-09-13T00:40:42.504 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: block_restart_interval: 16 2024-09-13T00:40:42.504 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: index_block_restart_interval: 1 2024-09-13T00:40:42.504 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: metadata_block_size: 4096 2024-09-13T00:40:42.504 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: partition_filters: 0 2024-09-13T00:40:42.504 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: use_delta_encoding: 1 2024-09-13T00:40:42.504 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: filter_policy: bloomfilter 2024-09-13T00:40:42.504 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: whole_key_filtering: 1 2024-09-13T00:40:42.504 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: verify_compression: 0 2024-09-13T00:40:42.504 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: read_amp_bytes_per_bit: 0 2024-09-13T00:40:42.504 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: format_version: 5 2024-09-13T00:40:42.504 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: enable_index_compression: 1 2024-09-13T00:40:42.504 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: block_align: 0 2024-09-13T00:40:42.504 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: max_auto_readahead_size: 262144 2024-09-13T00:40:42.505 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: prepopulate_block_cache: 0 2024-09-13T00:40:42.505 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: initial_auto_readahead_size: 8192 2024-09-13T00:40:42.505 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: num_file_reads_for_auto_readahead: 2 2024-09-13T00:40:42.505 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-13T00:40:42.505 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-13T00:40:42.505 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compression: NoCompression 2024-09-13T00:40:42.505 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-13T00:40:42.505 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-13T00:40:42.505 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T00:40:42.505 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.num_levels: 7 2024-09-13T00:40:42.505 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T00:40:42.505 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T00:40:42.506 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T00:40:42.506 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T00:40:42.506 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T00:40:42.506 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T00:40:42.506 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T00:40:42.506 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T00:40:42.506 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T00:40:42.506 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T00:40:42.506 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T00:40:42.506 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T00:40:42.506 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T00:40:42.506 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compression_opts.level: 32767 2024-09-13T00:40:42.507 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-13T00:40:42.507 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T00:40:42.507 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T00:40:42.507 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T00:40:42.507 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T00:40:42.507 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compression_opts.enabled: false 2024-09-13T00:40:42.507 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T00:40:42.507 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T00:40:42.507 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T00:40:42.507 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T00:40:42.507 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-13T00:40:42.507 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T00:40:42.508 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T00:40:42.508 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T00:40:42.508 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T00:40:42.508 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T00:40:42.508 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T00:40:42.508 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T00:40:42.508 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T00:40:42.508 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T00:40:42.508 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T00:40:42.508 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T00:40:42.508 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T00:40:42.508 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T00:40:42.508 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T00:40:42.509 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.arena_block_size: 1048576 2024-09-13T00:40:42.509 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T00:40:42.509 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T00:40:42.509 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-13T00:40:42.509 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T00:40:42.509 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T00:40:42.509 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T00:40:42.509 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T00:40:42.509 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T00:40:42.509 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T00:40:42.509 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T00:40:42.509 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T00:40:42.510 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T00:40:42.510 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T00:40:42.510 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T00:40:42.510 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.inplace_update_support: 0 2024-09-13T00:40:42.510 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T00:40:42.510 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T00:40:42.510 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T00:40:42.510 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T00:40:42.510 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.bloom_locality: 0 2024-09-13T00:40:42.510 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.max_successive_merges: 0 2024-09-13T00:40:42.510 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T00:40:42.510 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-13T00:40:42.511 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.force_consistency_checks: 1 2024-09-13T00:40:42.511 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-13T00:40:42.511 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.ttl: 2592000 2024-09-13T00:40:42.511 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T00:40:42.511 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T00:40:42.511 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T00:40:42.511 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.enable_blob_files: false 2024-09-13T00:40:42.511 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.min_blob_size: 0 2024-09-13T00:40:42.511 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.blob_file_size: 268435456 2024-09-13T00:40:42.511 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T00:40:42.511 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T00:40:42.511 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T00:40:42.511 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T00:40:42.512 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T00:40:42.512 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.blob_file_starting_level: 0 2024-09-13T00:40:42.512 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T00:40:42.512 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f997d0e5640 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 39.sst 2024-09-13T00:40:42.512 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T00:40:42.512 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 41, last_sequence is 21328, log_number is 37,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T00:40:42.512 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 37 2024-09-13T00:40:42.512 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: a5390e97-99af-4c74-8ca6-02517b6d2652 2024-09-13T00:40:42.512 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726188042305409, "job": 1, "event": "recovery_started", "wal_files": [37]} 2024-09-13T00:40:42.512 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.299+0000 7f998130eb00 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #37 mode 2 2024-09-13T00:40:42.512 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.315+0000 7f998130eb00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726188042322027, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 42, "file_size": 3600867, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 21334, "largest_seqno": 23608, "table_properties": {"data_size": 3589783, "index_size": 7926, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2117, "raw_key_size": 18546, "raw_average_key_size": 22, "raw_value_size": 3570001, "raw_average_value_size": 4296, "num_data_blocks": 344, "num_entries": 831, "num_filter_entries": 831, "num_deletions": 4, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726188042, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "a5390e97-99af-4c74-8ca6-02517b6d2652", "db_session_id": "455VXH5NT5NKZ538NZ7S", "orig_file_number": 42, "seqno_to_time_mapping": "N/A"}} 2024-09-13T00:40:42.512 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.315+0000 7f998130eb00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726188042322173, "job": 1, "event": "recovery_finished"} 2024-09-13T00:40:42.513 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.315+0000 7f998130eb00 4 rocksdb: [db/version_set.cc:5047] Creating manifest 44 2024-09-13T00:40:42.513 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.315+0000 7f998130eb00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T00:40:42.513 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.319+0000 7f998130eb00 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000037.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T00:40:42.513 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.319+0000 7f998130eb00 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x562566cb4e00 2024-09-13T00:40:42.513 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.319+0000 7f998130eb00 4 rocksdb: DB pointer 0x562566dac000 2024-09-13T00:40:42.513 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.319+0000 7f99770d9640 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T00:40:42.513 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.319+0000 7f99770d9640 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T00:40:42.513 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: ** DB Stats ** 2024-09-13T00:40:42.513 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:40:42.513 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T00:40:42.513 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T00:40:42.513 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T00:40:42.513 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T00:40:42.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T00:40:42.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T00:40:42.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: ** Compaction Stats [default] ** 2024-09-13T00:40:42.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T00:40:42.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T00:40:42.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: L0 1/0 3.43 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 463.0 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-13T00:40:42.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: L6 1/0 11.16 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T00:40:42.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Sum 2/0 14.60 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 463.0 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-13T00:40:42.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 463.0 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-13T00:40:42.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: ** Compaction Stats [default] ** 2024-09-13T00:40:42.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T00:40:42.514 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:40:42.515 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 463.0 0.01 0.00 1 0.007 0 0 0.0 0.0 2024-09-13T00:40:42.515 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T00:40:42.515 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:40:42.515 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Flush(GB): cumulative 0.003, interval 0.003 2024-09-13T00:40:42.515 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T00:40:42.515 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T00:40:42.515 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T00:40:42.515 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T00:40:42.515 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Cumulative compaction: 0.00 GB write, 172.79 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:40:42.515 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Interval compaction: 0.00 GB write, 172.79 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:40:42.515 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T00:40:42.515 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Block cache BinnedLRUCache@0x562566c8b350#7 capacity: 512.00 MB usage: 10.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.7e-05 secs_since: 0 2024-09-13T00:40:42.516 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Block cache entry stats(count,size,portion): FilterBlock(1,2.28 KB,0.000435114%) IndexBlock(1,8.08 KB,0.00154078%) Misc(1,0.00 KB,0%) 2024-09-13T00:40:42.516 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T00:40:42.516 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.319+0000 7f998130eb00 0 starting mon.c rank 2 at public addrs [v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0] at bind addrs [v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:40:42.516 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.319+0000 7f998130eb00 1 mon.c@-1(???) e3 preinit fsid 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:40:42.516 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.319+0000 7f998130eb00 0 mon.c@-1(???).mds e9 new map 2024-09-13T00:40:42.516 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.319+0000 7f998130eb00 0 mon.c@-1(???).mds e9 print_map 2024-09-13T00:40:42.516 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: e9 2024-09-13T00:40:42.516 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T00:40:42.516 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T00:40:42.516 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: legacy client fscid: 1 2024-09-13T00:40:42.516 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: 2024-09-13T00:40:42.516 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Filesystem 'foo' (1) 2024-09-13T00:40:42.516 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: fs_name foo 2024-09-13T00:40:42.517 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: epoch 9 2024-09-13T00:40:42.517 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T00:40:42.517 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: created 2024-09-13T00:32:23.134111+0000 2024-09-13T00:40:42.517 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: modified 2024-09-13T00:32:33.870108+0000 2024-09-13T00:40:42.517 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: tableserver 0 2024-09-13T00:40:42.517 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: root 0 2024-09-13T00:40:42.517 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: session_timeout 60 2024-09-13T00:40:42.517 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: session_autoclose 300 2024-09-13T00:40:42.517 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: max_file_size 1099511627776 2024-09-13T00:40:42.517 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: max_xattr_size 65536 2024-09-13T00:40:42.517 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: required_client_features {} 2024-09-13T00:40:42.517 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: last_failure 0 2024-09-13T00:40:42.517 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: last_failure_osd_epoch 63 2024-09-13T00:40:42.518 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T00:40:42.518 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: max_mds 1 2024-09-13T00:40:42.518 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: in 0 2024-09-13T00:40:42.518 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: up {0=24349} 2024-09-13T00:40:42.518 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: failed 2024-09-13T00:40:42.518 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: damaged 2024-09-13T00:40:42.518 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: stopped 2024-09-13T00:40:42.518 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: data_pools [4] 2024-09-13T00:40:42.518 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: metadata_pool 3 2024-09-13T00:40:42.518 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: inline_data disabled 2024-09-13T00:40:42.518 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: balancer 2024-09-13T00:40:42.518 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: bal_rank_mask -1 2024-09-13T00:40:42.518 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: standby_count_wanted 1 2024-09-13T00:40:42.519 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: [mds.foo.smithi088.krmowh{0:24349} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.88:6832/1144032959,v1:172.21.15.88:6833/1144032959] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T00:40:42.519 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: 2024-09-13T00:40:42.519 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: 2024-09-13T00:40:42.519 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: Standby daemons: 2024-09-13T00:40:42.519 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: 2024-09-13T00:40:42.519 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: [mds.foo.smithi077.bigbbf{-1:14604} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.77:6834/3765041927,v1:172.21.15.77:6835/3765041927] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T00:40:42.519 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.319+0000 7f998130eb00 0 mon.c@-1(???).osd e378 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T00:40:42.519 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.319+0000 7f998130eb00 0 mon.c@-1(???).osd e378 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T00:40:42.519 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.319+0000 7f998130eb00 0 mon.c@-1(???).osd e378 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T00:40:42.519 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.319+0000 7f998130eb00 0 mon.c@-1(???).osd e378 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T00:40:42.519 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:42 smithi077 bash[83127]: debug 2024-09-13T00:40:42.323+0000 7f998130eb00 1 mon.c@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2024-09-13T00:40:42.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:42 smithi088 bash[61791]: ts=2024-09-13T00:40:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:43.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:43 smithi088 bash[61791]: ts=2024-09-13T00:40:43.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:44.627 INFO:teuthology.orchestra.run.smithi077.stdout:wait for majority of mons to upgrade 2024-09-13T00:40:44.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:44 smithi088 bash[61791]: ts=2024-09-13T00:40:44.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:44.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:44 smithi088 bash[61791]: ts=2024-09-13T00:40:44.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:45.555 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-09-13T00:40:48.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:47 smithi077 bash[80681]: debug 2024-09-13T00:40:47.586+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-09-13T00:40:48.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:47 smithi077 bash[80681]: debug 2024-09-13T00:40:47.586+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-13T00:40:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:47 smithi077 bash[80681]: debug 2024-09-13T00:40:47.586+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : mon.c (rank 2) addr [v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0] is down (out of quorum) 2024-09-13T00:40:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:47 smithi077 bash[80681]: debug 2024-09-13T00:40:47.586+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:40:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:47 smithi077 bash[80681]: debug 2024-09-13T00:40:47.586+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:47 smithi077 bash[80681]: debug 2024-09-13T00:40:47.586+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:47 smithi077 bash[80681]: debug 2024-09-13T00:40:47.586+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:47 smithi077 bash[80681]: debug 2024-09-13T00:40:47.586+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:47 smithi077 bash[80681]: debug 2024-09-13T00:40:47.586+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:47 smithi077 bash[80681]: debug 2024-09-13T00:40:47.586+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:47 smithi077 bash[80681]: debug 2024-09-13T00:40:47.586+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:47 smithi077 bash[80681]: debug 2024-09-13T00:40:47.586+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:40:48] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.43.0" 2024-09-13T00:40:48.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:40.364613+0000 mgr.y (mgr.25304) 73 : cluster [DBG] pgmap v142: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.0 MiB/s wr, 95 op/s 2024-09-13T00:40:48.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:42.365079+0000 mgr.y (mgr.25304) 74 : cluster [DBG] pgmap v143: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.5 MiB/s wr, 78 op/s 2024-09-13T00:40:48.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:42.536882+0000 mon.b (mon.1) 141 : cluster [INF] mon.b calling monitor election 2024-09-13T00:40:48.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:42.542412+0000 mon.a (mon.0) 174 : cluster [INF] mon.a calling monitor election 2024-09-13T00:40:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: audit 2024-09-13T00:40:43.409610+0000 mon.b (mon.1) 142 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:44.366294+0000 mgr.y (mgr.25304) 75 : cluster [DBG] pgmap v144: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 64 op/s 2024-09-13T00:40:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:46.366995+0000 mgr.y (mgr.25304) 76 : cluster [DBG] pgmap v145: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 MiB/s wr, 54 op/s 2024-09-13T00:40:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.543797+0000 mon.a (mon.0) 175 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T00:40:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.552481+0000 mon.a (mon.0) 176 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:40:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.553621+0000 mon.a (mon.0) 177 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:40:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.553676+0000 mon.a (mon.0) 178 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2024-09-13T00:40:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.554247+0000 mon.a (mon.0) 179 : cluster [DBG] mgrmap e39: y(active, since 109s), standbys: x 2024-09-13T00:40:48.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.554567+0000 mon.a (mon.0) 180 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-13T00:40:48.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.591948+0000 mon.a (mon.0) 181 : cluster [ERR] Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-09-13T00:40:48.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.591983+0000 mon.a (mon.0) 182 : cluster [ERR] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-13T00:40:48.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.591999+0000 mon.a (mon.0) 183 : cluster [ERR] mon.c (rank 2) addr [v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0] is down (out of quorum) 2024-09-13T00:40:48.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.592015+0000 mon.a (mon.0) 184 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:40:48.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.592030+0000 mon.a (mon.0) 185 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.592046+0000 mon.a (mon.0) 186 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.592064+0000 mon.a (mon.0) 187 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:40.364613+0000 mgr.y (mgr.25304) 73 : cluster [DBG] pgmap v142: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.0 MiB/s wr, 95 op/s 2024-09-13T00:40:48.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:42.365079+0000 mgr.y (mgr.25304) 74 : cluster [DBG] pgmap v143: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.5 MiB/s wr, 78 op/s 2024-09-13T00:40:48.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:42.536882+0000 mon.b (mon.1) 141 : cluster [INF] mon.b calling monitor election 2024-09-13T00:40:48.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:42.542412+0000 mon.a (mon.0) 174 : cluster [INF] mon.a calling monitor election 2024-09-13T00:40:48.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: audit 2024-09-13T00:40:43.409610+0000 mon.b (mon.1) 142 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:48.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:44.366294+0000 mgr.y (mgr.25304) 75 : cluster [DBG] pgmap v144: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 64 op/s 2024-09-13T00:40:48.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:46.366995+0000 mgr.y (mgr.25304) 76 : cluster [DBG] pgmap v145: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 MiB/s wr, 54 op/s 2024-09-13T00:40:48.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.543797+0000 mon.a (mon.0) 175 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T00:40:48.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.552481+0000 mon.a (mon.0) 176 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:40:48.783 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.553621+0000 mon.a (mon.0) 177 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:40:48.783 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.553676+0000 mon.a (mon.0) 178 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2024-09-13T00:40:48.783 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.554247+0000 mon.a (mon.0) 179 : cluster [DBG] mgrmap e39: y(active, since 109s), standbys: x 2024-09-13T00:40:48.783 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.554567+0000 mon.a (mon.0) 180 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-13T00:40:48.784 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.591948+0000 mon.a (mon.0) 181 : cluster [ERR] Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-09-13T00:40:48.784 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.591983+0000 mon.a (mon.0) 182 : cluster [ERR] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-13T00:40:48.784 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.591999+0000 mon.a (mon.0) 183 : cluster [ERR] mon.c (rank 2) addr [v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0] is down (out of quorum) 2024-09-13T00:40:48.784 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.592015+0000 mon.a (mon.0) 184 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:40:48.784 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.592030+0000 mon.a (mon.0) 185 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.784 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.592046+0000 mon.a (mon.0) 186 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.785 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.592064+0000 mon.a (mon.0) 187 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.785 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.592107+0000 mon.a (mon.0) 188 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.785 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.592145+0000 mon.a (mon.0) 189 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.785 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.592183+0000 mon.a (mon.0) 190 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.786 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.592215+0000 mon.a (mon.0) 191 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.786 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:47.592242+0000 mon.a (mon.0) 192 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.786 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: audit 2024-09-13T00:40:47.599029+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:48.787 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: audit 2024-09-13T00:40:47.605384+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:48.788 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: audit 2024-09-13T00:40:47.616665+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:48.788 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: audit 2024-09-13T00:40:47.620312+0000 mon.b (mon.1) 143 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:48.789 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: audit 2024-09-13T00:40:48.016732+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:48.789 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: audit 2024-09-13T00:40:48.021767+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:48.789 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.367539+0000 mgr.y (mgr.25304) 77 : cluster [DBG] pgmap v146: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 1.5 MiB/s wr, 49 op/s 2024-09-13T00:40:48.789 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.592107+0000 mon.a (mon.0) 188 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.789 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.592145+0000 mon.a (mon.0) 189 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.790 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.592183+0000 mon.a (mon.0) 190 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.790 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.592215+0000 mon.a (mon.0) 191 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.790 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:47.592242+0000 mon.a (mon.0) 192 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.790 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: audit 2024-09-13T00:40:47.599029+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:48.790 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: audit 2024-09-13T00:40:47.605384+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:48.790 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: audit 2024-09-13T00:40:47.616665+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:48.790 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: audit 2024-09-13T00:40:47.620312+0000 mon.b (mon.1) 143 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:48.790 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: audit 2024-09-13T00:40:48.016732+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:48.790 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: audit 2024-09-13T00:40:48.021767+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:48.790 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.367539+0000 mgr.y (mgr.25304) 77 : cluster [DBG] pgmap v146: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 1.5 MiB/s wr, 49 op/s 2024-09-13T00:40:48.790 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: debug 2024-09-13T00:40:48.606+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-09-13T00:40:48.790 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: debug 2024-09-13T00:40:48.606+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:40:48.790 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: debug 2024-09-13T00:40:48.606+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.791 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: debug 2024-09-13T00:40:48.606+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.791 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: debug 2024-09-13T00:40:48.606+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.791 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: debug 2024-09-13T00:40:48.606+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.791 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: debug 2024-09-13T00:40:48.606+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.791 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: debug 2024-09-13T00:40:48.606+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.791 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: debug 2024-09-13T00:40:48.606+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.791 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:48 smithi077 bash[80681]: debug 2024-09-13T00:40:48.606+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.791 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.SnapDiscard (11452 ms) 2024-09-13T00:40:48.791 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.CloneDiscard 2024-09-13T00:40:48.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[61791]: ts=2024-09-13T00:40:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:40.364613+0000 mgr.y (mgr.25304) 73 : cluster [DBG] pgmap v142: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 3.0 MiB/s wr, 95 op/s 2024-09-13T00:40:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:42.365079+0000 mgr.y (mgr.25304) 74 : cluster [DBG] pgmap v143: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 15 MiB/s rd, 2.5 MiB/s wr, 78 op/s 2024-09-13T00:40:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:42.536882+0000 mon.b (mon.1) 141 : cluster [INF] mon.b calling monitor election 2024-09-13T00:40:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:42.542412+0000 mon.a (mon.0) 174 : cluster [INF] mon.a calling monitor election 2024-09-13T00:40:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: audit 2024-09-13T00:40:43.409610+0000 mon.b (mon.1) 142 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:44.366294+0000 mgr.y (mgr.25304) 75 : cluster [DBG] pgmap v144: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 12 MiB/s rd, 2.0 MiB/s wr, 64 op/s 2024-09-13T00:40:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:46.366995+0000 mgr.y (mgr.25304) 76 : cluster [DBG] pgmap v145: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 10 MiB/s rd, 1.7 MiB/s wr, 54 op/s 2024-09-13T00:40:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.543797+0000 mon.a (mon.0) 175 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T00:40:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.552481+0000 mon.a (mon.0) 176 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:40:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.553621+0000 mon.a (mon.0) 177 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:40:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.553676+0000 mon.a (mon.0) 178 : cluster [DBG] osdmap e378: 8 total, 8 up, 8 in 2024-09-13T00:40:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.554247+0000 mon.a (mon.0) 179 : cluster [DBG] mgrmap e39: y(active, since 109s), standbys: x 2024-09-13T00:40:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.554567+0000 mon.a (mon.0) 180 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-13T00:40:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.591948+0000 mon.a (mon.0) 181 : cluster [ERR] Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-09-13T00:40:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.591983+0000 mon.a (mon.0) 182 : cluster [ERR] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-13T00:40:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.591999+0000 mon.a (mon.0) 183 : cluster [ERR] mon.c (rank 2) addr [v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0] is down (out of quorum) 2024-09-13T00:40:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.592015+0000 mon.a (mon.0) 184 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:40:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.592030+0000 mon.a (mon.0) 185 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.592046+0000 mon.a (mon.0) 186 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.592064+0000 mon.a (mon.0) 187 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.592107+0000 mon.a (mon.0) 188 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.592145+0000 mon.a (mon.0) 189 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.592183+0000 mon.a (mon.0) 190 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.592215+0000 mon.a (mon.0) 191 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:47.592242+0000 mon.a (mon.0) 192 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:48.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: audit 2024-09-13T00:40:47.599029+0000 mon.a (mon.0) 193 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:48.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: audit 2024-09-13T00:40:47.605384+0000 mon.a (mon.0) 194 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:48.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: audit 2024-09-13T00:40:47.616665+0000 mon.a (mon.0) 195 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:48.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: audit 2024-09-13T00:40:47.620312+0000 mon.b (mon.1) 143 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:40:48.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: audit 2024-09-13T00:40:48.016732+0000 mon.a (mon.0) 196 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:48.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: audit 2024-09-13T00:40:48.021767+0000 mon.a (mon.0) 197 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:48.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:48 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.367539+0000 mgr.y (mgr.25304) 77 : cluster [DBG] pgmap v146: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 9.3 MiB/s rd, 1.5 MiB/s wr, 49 op/s 2024-09-13T00:40:49.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:42.536507+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-13T00:40:49.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.568530+0000 mon.b (mon.1) 144 : cluster [INF] mon.b calling monitor election 2024-09-13T00:40:49.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.571038+0000 mon.a (mon.0) 198 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2024-09-13T00:40:49.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.572332+0000 mon.a (mon.0) 199 : cluster [INF] mon.a calling monitor election 2024-09-13T00:40:49.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.586970+0000 mon.a (mon.0) 200 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T00:40:49.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.596319+0000 mon.a (mon.0) 201 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:40:49.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.597721+0000 mon.a (mon.0) 202 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:40:49.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.597837+0000 mon.a (mon.0) 203 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2024-09-13T00:40:49.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.599213+0000 mon.a (mon.0) 204 : cluster [DBG] mgrmap e39: y(active, since 110s), standbys: x 2024-09-13T00:40:49.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.599651+0000 mon.a (mon.0) 205 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T00:40:49.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.611004+0000 mon.a (mon.0) 206 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-09-13T00:40:49.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.611057+0000 mon.a (mon.0) 207 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:40:49.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.611095+0000 mon.a (mon.0) 208 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:49.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.611142+0000 mon.a (mon.0) 209 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:49.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.611177+0000 mon.a (mon.0) 210 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:49.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.611211+0000 mon.a (mon.0) 211 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:49.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.611239+0000 mon.a (mon.0) 212 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:49.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.611265+0000 mon.a (mon.0) 213 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:49.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.611292+0000 mon.a (mon.0) 214 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:49.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:49 smithi088 bash[24895]: cluster 2024-09-13T00:40:48.611326+0000 mon.a (mon.0) 215 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.009 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.CloneDiscard (1218 ms) 2024-09-13T00:40:50.009 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.CloneShrink 2024-09-13T00:40:50.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:42.536507+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-13T00:40:50.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.568530+0000 mon.b (mon.1) 144 : cluster [INF] mon.b calling monitor election 2024-09-13T00:40:50.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.571038+0000 mon.a (mon.0) 198 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2024-09-13T00:40:50.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.572332+0000 mon.a (mon.0) 199 : cluster [INF] mon.a calling monitor election 2024-09-13T00:40:50.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.586970+0000 mon.a (mon.0) 200 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T00:40:50.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.596319+0000 mon.a (mon.0) 201 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:40:50.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.597721+0000 mon.a (mon.0) 202 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:40:50.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.597837+0000 mon.a (mon.0) 203 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2024-09-13T00:40:50.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.599213+0000 mon.a (mon.0) 204 : cluster [DBG] mgrmap e39: y(active, since 110s), standbys: x 2024-09-13T00:40:50.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.599651+0000 mon.a (mon.0) 205 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T00:40:50.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.611004+0000 mon.a (mon.0) 206 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-09-13T00:40:50.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.611057+0000 mon.a (mon.0) 207 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:40:50.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.611095+0000 mon.a (mon.0) 208 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.611142+0000 mon.a (mon.0) 209 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.611177+0000 mon.a (mon.0) 210 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.611211+0000 mon.a (mon.0) 211 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.611239+0000 mon.a (mon.0) 212 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.611265+0000 mon.a (mon.0) 213 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.611292+0000 mon.a (mon.0) 214 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[80681]: cluster 2024-09-13T00:40:48.611326+0000 mon.a (mon.0) 215 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:42.536507+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-09-13T00:40:50.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.568530+0000 mon.b (mon.1) 144 : cluster [INF] mon.b calling monitor election 2024-09-13T00:40:50.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.571038+0000 mon.a (mon.0) 198 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2024-09-13T00:40:50.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.572332+0000 mon.a (mon.0) 199 : cluster [INF] mon.a calling monitor election 2024-09-13T00:40:50.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.586970+0000 mon.a (mon.0) 200 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T00:40:50.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.596319+0000 mon.a (mon.0) 201 : cluster [DBG] monmap e3: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:40:50.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.597721+0000 mon.a (mon.0) 202 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:40:50.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.597837+0000 mon.a (mon.0) 203 : cluster [DBG] osdmap e379: 8 total, 8 up, 8 in 2024-09-13T00:40:50.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.599213+0000 mon.a (mon.0) 204 : cluster [DBG] mgrmap e39: y(active, since 110s), standbys: x 2024-09-13T00:40:50.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.599651+0000 mon.a (mon.0) 205 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T00:40:50.031 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.611004+0000 mon.a (mon.0) 206 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-09-13T00:40:50.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.611057+0000 mon.a (mon.0) 207 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:40:50.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.611095+0000 mon.a (mon.0) 208 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.611142+0000 mon.a (mon.0) 209 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.611177+0000 mon.a (mon.0) 210 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.611211+0000 mon.a (mon.0) 211 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.611239+0000 mon.a (mon.0) 212 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.611265+0000 mon.a (mon.0) 213 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.611292+0000 mon.a (mon.0) 214 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.032 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:49 smithi077 bash[83127]: cluster 2024-09-13T00:40:48.611326+0000 mon.a (mon.0) 215 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:40:50.351 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:50 smithi088 bash[61791]: ts=2024-09-13T00:40:50.088Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", public_addr=\"172.21.15.77\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", public_addr=\"172.21.15.77\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-09-13T00:40:50.352 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:40:50.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:50 smithi077 bash[83127]: cluster 2024-09-13T00:40:49.615060+0000 mon.a (mon.0) 216 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2024-09-13T00:40:50.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:50 smithi077 bash[83127]: cluster 2024-09-13T00:40:50.367967+0000 mgr.y (mgr.25304) 78 : cluster [DBG] pgmap v149: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 2.1 KiB/s wr, 18 op/s 2024-09-13T00:40:50.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:50 smithi077 bash[80681]: cluster 2024-09-13T00:40:49.615060+0000 mon.a (mon.0) 216 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2024-09-13T00:40:50.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:50 smithi077 bash[80681]: cluster 2024-09-13T00:40:50.367967+0000 mgr.y (mgr.25304) 78 : cluster [DBG] pgmap v149: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 2.1 KiB/s wr, 18 op/s 2024-09-13T00:40:51.027 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.CloneShrink (1018 ms) 2024-09-13T00:40:51.027 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.CloneExpand 2024-09-13T00:40:51.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:50 smithi088 bash[24895]: cluster 2024-09-13T00:40:49.615060+0000 mon.a (mon.0) 216 : cluster [DBG] osdmap e380: 8 total, 8 up, 8 in 2024-09-13T00:40:51.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:50 smithi088 bash[24895]: cluster 2024-09-13T00:40:50.367967+0000 mgr.y (mgr.25304) 78 : cluster [DBG] pgmap v149: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 11 KiB/s rd, 2.1 KiB/s wr, 18 op/s 2024-09-13T00:40:51.900 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:51 smithi077 bash[80681]: cluster 2024-09-13T00:40:50.626002+0000 mon.a (mon.0) 217 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2024-09-13T00:40:51.900 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:51 smithi077 bash[83127]: cluster 2024-09-13T00:40:50.626002+0000 mon.a (mon.0) 217 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2024-09-13T00:40:52.024 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.CloneExpand (997 ms) 2024-09-13T00:40:52.025 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.CloneHideParent 2024-09-13T00:40:52.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:51 smithi088 bash[24895]: cluster 2024-09-13T00:40:50.626002+0000 mon.a (mon.0) 217 : cluster [DBG] osdmap e381: 8 total, 8 up, 8 in 2024-09-13T00:40:52.527 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:40:52 smithi077 bash[73137]: debug 2024-09-13T00:40:52.330+0000 7ffa94da1640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-13T00:40:52.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:52 smithi088 bash[61791]: ts=2024-09-13T00:40:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:53.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:52 smithi077 bash[80681]: cluster 2024-09-13T00:40:51.629163+0000 mon.a (mon.0) 218 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2024-09-13T00:40:53.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:52 smithi077 bash[80681]: cluster 2024-09-13T00:40:52.368598+0000 mgr.y (mgr.25304) 79 : cluster [DBG] pgmap v152: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 KiB/s wr, 36 op/s 2024-09-13T00:40:53.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:52 smithi077 bash[83127]: cluster 2024-09-13T00:40:51.629163+0000 mon.a (mon.0) 218 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2024-09-13T00:40:53.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:52 smithi077 bash[83127]: cluster 2024-09-13T00:40:52.368598+0000 mgr.y (mgr.25304) 79 : cluster [DBG] pgmap v152: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 KiB/s wr, 36 op/s 2024-09-13T00:40:53.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:52 smithi088 bash[24895]: cluster 2024-09-13T00:40:51.629163+0000 mon.a (mon.0) 218 : cluster [DBG] osdmap e382: 8 total, 8 up, 8 in 2024-09-13T00:40:53.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:52 smithi088 bash[24895]: cluster 2024-09-13T00:40:52.368598+0000 mgr.y (mgr.25304) 79 : cluster [DBG] pgmap v152: 105 pgs: 105 active+clean; 65 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 4.2 KiB/s wr, 36 op/s 2024-09-13T00:40:53.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:53 smithi088 bash[24895]: cluster 2024-09-13T00:40:52.648075+0000 mon.a (mon.0) 219 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2024-09-13T00:40:53.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:53 smithi088 bash[24895]: audit 2024-09-13T00:40:53.065491+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:53.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:53 smithi088 bash[24895]: audit 2024-09-13T00:40:53.077924+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:53.852 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:53 smithi088 bash[61791]: ts=2024-09-13T00:40:53.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:54.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:53 smithi077 bash[80681]: cluster 2024-09-13T00:40:52.648075+0000 mon.a (mon.0) 219 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2024-09-13T00:40:54.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:53 smithi077 bash[80681]: audit 2024-09-13T00:40:53.065491+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:54.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:53 smithi077 bash[80681]: audit 2024-09-13T00:40:53.077924+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:54.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:53 smithi077 bash[83127]: cluster 2024-09-13T00:40:52.648075+0000 mon.a (mon.0) 219 : cluster [DBG] osdmap e383: 8 total, 8 up, 8 in 2024-09-13T00:40:54.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:53 smithi077 bash[83127]: audit 2024-09-13T00:40:53.065491+0000 mon.a (mon.0) 220 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:54.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:53 smithi077 bash[83127]: audit 2024-09-13T00:40:53.077924+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:54.464 INFO:teuthology.orchestra.run.smithi077.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-13T00:40:54.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:54 smithi077 bash[80681]: cluster 2024-09-13T00:40:53.660868+0000 mon.a (mon.0) 222 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2024-09-13T00:40:54.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:54 smithi077 bash[80681]: audit 2024-09-13T00:40:53.744536+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:54.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:54 smithi077 bash[80681]: audit 2024-09-13T00:40:53.754079+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:54.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:54 smithi077 bash[80681]: cluster 2024-09-13T00:40:54.369804+0000 mgr.y (mgr.25304) 80 : cluster [DBG] pgmap v155: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-13T00:40:54.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:54 smithi077 bash[80681]: audit 2024-09-13T00:40:54.456841+0000 mon.a (mon.0) 225 : audit [DBG] from='client.? 172.21.15.77:0/3760835027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:40:54.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:54 smithi077 bash[83127]: cluster 2024-09-13T00:40:53.660868+0000 mon.a (mon.0) 222 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2024-09-13T00:40:54.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:54 smithi077 bash[83127]: audit 2024-09-13T00:40:53.744536+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:54.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:54 smithi077 bash[83127]: audit 2024-09-13T00:40:53.754079+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:54.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:54 smithi077 bash[83127]: cluster 2024-09-13T00:40:54.369804+0000 mgr.y (mgr.25304) 80 : cluster [DBG] pgmap v155: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-13T00:40:54.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:54 smithi077 bash[83127]: audit 2024-09-13T00:40:54.456841+0000 mon.a (mon.0) 225 : audit [DBG] from='client.? 172.21.15.77:0/3760835027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:40:54.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:54 smithi088 bash[61791]: ts=2024-09-13T00:40:54.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:54.852 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:54 smithi088 bash[61791]: ts=2024-09-13T00:40:54.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:54.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:54 smithi088 bash[24895]: cluster 2024-09-13T00:40:53.660868+0000 mon.a (mon.0) 222 : cluster [DBG] osdmap e384: 8 total, 8 up, 8 in 2024-09-13T00:40:54.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:54 smithi088 bash[24895]: audit 2024-09-13T00:40:53.744536+0000 mon.a (mon.0) 223 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:54.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:54 smithi088 bash[24895]: audit 2024-09-13T00:40:53.754079+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:40:54.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:54 smithi088 bash[24895]: cluster 2024-09-13T00:40:54.369804+0000 mgr.y (mgr.25304) 80 : cluster [DBG] pgmap v155: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-13T00:40:54.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:54 smithi088 bash[24895]: audit 2024-09-13T00:40:54.456841+0000 mon.a (mon.0) 225 : audit [DBG] from='client.? 172.21.15.77:0/3760835027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:40:55.391 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph orch ps' 2024-09-13T00:40:56.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:55 smithi077 bash[80681]: cluster 2024-09-13T00:40:54.669725+0000 mon.a (mon.0) 226 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2024-09-13T00:40:56.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:55 smithi077 bash[83127]: cluster 2024-09-13T00:40:54.669725+0000 mon.a (mon.0) 226 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2024-09-13T00:40:56.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:55 smithi088 bash[24895]: cluster 2024-09-13T00:40:54.669725+0000 mon.a (mon.0) 226 : cluster [DBG] osdmap e385: 8 total, 8 up, 8 in 2024-09-13T00:40:57.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:56 smithi077 bash[80681]: cluster 2024-09-13T00:40:55.680242+0000 mon.a (mon.0) 227 : cluster [DBG] osdmap e386: 8 total, 8 up, 8 in 2024-09-13T00:40:57.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:56 smithi077 bash[80681]: cluster 2024-09-13T00:40:56.370317+0000 mgr.y (mgr.25304) 81 : cluster [DBG] pgmap v158: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-13T00:40:57.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:56 smithi077 bash[83127]: cluster 2024-09-13T00:40:55.680242+0000 mon.a (mon.0) 227 : cluster [DBG] osdmap e386: 8 total, 8 up, 8 in 2024-09-13T00:40:57.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:56 smithi077 bash[83127]: cluster 2024-09-13T00:40:56.370317+0000 mgr.y (mgr.25304) 81 : cluster [DBG] pgmap v158: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-13T00:40:57.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:56 smithi088 bash[24895]: cluster 2024-09-13T00:40:55.680242+0000 mon.a (mon.0) 227 : cluster [DBG] osdmap e386: 8 total, 8 up, 8 in 2024-09-13T00:40:57.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:56 smithi088 bash[24895]: cluster 2024-09-13T00:40:56.370317+0000 mgr.y (mgr.25304) 81 : cluster [DBG] pgmap v158: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-13T00:40:58.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:57 smithi077 bash[80681]: cluster 2024-09-13T00:40:56.689175+0000 mon.a (mon.0) 228 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2024-09-13T00:40:58.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:57 smithi077 bash[83127]: cluster 2024-09-13T00:40:56.689175+0000 mon.a (mon.0) 228 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2024-09-13T00:40:58.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:57 smithi088 bash[24895]: cluster 2024-09-13T00:40:56.689175+0000 mon.a (mon.0) 228 : cluster [DBG] osdmap e387: 8 total, 8 up, 8 in 2024-09-13T00:40:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:40:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:40:58] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.43.0" 2024-09-13T00:40:58.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:58 smithi077 bash[80681]: cluster 2024-09-13T00:40:57.704568+0000 mon.a (mon.0) 229 : cluster [DBG] osdmap e388: 8 total, 8 up, 8 in 2024-09-13T00:40:58.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:58 smithi077 bash[80681]: cluster 2024-09-13T00:40:58.370916+0000 mgr.y (mgr.25304) 82 : cluster [DBG] pgmap v161: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:40:58.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:58 smithi077 bash[80681]: audit 2024-09-13T00:40:58.410370+0000 mon.b (mon.1) 145 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:58.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:58 smithi077 bash[83127]: cluster 2024-09-13T00:40:57.704568+0000 mon.a (mon.0) 229 : cluster [DBG] osdmap e388: 8 total, 8 up, 8 in 2024-09-13T00:40:58.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:58 smithi077 bash[83127]: cluster 2024-09-13T00:40:58.370916+0000 mgr.y (mgr.25304) 82 : cluster [DBG] pgmap v161: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:40:58.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:58 smithi077 bash[83127]: audit 2024-09-13T00:40:58.410370+0000 mon.b (mon.1) 145 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:58.938 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.CloneHideParent (6914 ms) 2024-09-13T00:40:58.938 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.Clone 2024-09-13T00:40:59.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:58 smithi088 bash[24895]: cluster 2024-09-13T00:40:57.704568+0000 mon.a (mon.0) 229 : cluster [DBG] osdmap e388: 8 total, 8 up, 8 in 2024-09-13T00:40:59.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:58 smithi088 bash[24895]: cluster 2024-09-13T00:40:58.370916+0000 mgr.y (mgr.25304) 82 : cluster [DBG] pgmap v161: 105 pgs: 105 active+clean; 65 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:40:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:58 smithi088 bash[24895]: audit 2024-09-13T00:40:58.410370+0000 mon.b (mon.1) 145 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:40:59.102 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:40:58 smithi088 bash[61791]: ts=2024-09-13T00:40:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:40:59.945 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:41:00.027 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:40:59 smithi077 bash[83127]: cluster 2024-09-13T00:40:58.708353+0000 mon.a (mon.0) 230 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2024-09-13T00:41:00.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:40:59 smithi077 bash[80681]: cluster 2024-09-13T00:40:58.708353+0000 mon.a (mon.0) 230 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2024-09-13T00:41:00.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:40:59 smithi088 bash[24895]: cluster 2024-09-13T00:40:58.708353+0000 mon.a (mon.0) 230 : cluster [DBG] osdmap e389: 8 total, 8 up, 8 in 2024-09-13T00:41:01.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:00 smithi077 bash[80681]: cluster 2024-09-13T00:40:59.712122+0000 mon.a (mon.0) 231 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2024-09-13T00:41:01.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:00 smithi077 bash[80681]: cluster 2024-09-13T00:41:00.371637+0000 mgr.y (mgr.25304) 83 : cluster [DBG] pgmap v164: 105 pgs: 105 active+clean; 81 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 4.0 MiB/s wr, 175 op/s 2024-09-13T00:41:01.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:00 smithi077 bash[83127]: cluster 2024-09-13T00:40:59.712122+0000 mon.a (mon.0) 231 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2024-09-13T00:41:01.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:00 smithi077 bash[83127]: cluster 2024-09-13T00:41:00.371637+0000 mgr.y (mgr.25304) 83 : cluster [DBG] pgmap v164: 105 pgs: 105 active+clean; 81 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 4.0 MiB/s wr, 175 op/s 2024-09-13T00:41:01.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:00 smithi088 bash[24895]: cluster 2024-09-13T00:40:59.712122+0000 mon.a (mon.0) 231 : cluster [DBG] osdmap e390: 8 total, 8 up, 8 in 2024-09-13T00:41:01.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:00 smithi088 bash[24895]: cluster 2024-09-13T00:41:00.371637+0000 mgr.y (mgr.25304) 83 : cluster [DBG] pgmap v164: 105 pgs: 105 active+clean; 81 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 4.0 MiB/s wr, 175 op/s 2024-09-13T00:41:01.361 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.Clone (2423 ms) 2024-09-13T00:41:01.361 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.CloneFlatten 2024-09-13T00:41:01.900 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:01 smithi077 bash[83127]: cluster 2024-09-13T00:41:00.735089+0000 mon.a (mon.0) 232 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2024-09-13T00:41:01.901 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:01 smithi077 bash[80681]: cluster 2024-09-13T00:41:00.735089+0000 mon.a (mon.0) 232 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2024-09-13T00:41:02.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:01 smithi088 bash[24895]: cluster 2024-09-13T00:41:00.735089+0000 mon.a (mon.0) 232 : cluster [DBG] osdmap e391: 8 total, 8 up, 8 in 2024-09-13T00:41:02.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:02 smithi088 bash[61791]: ts=2024-09-13T00:41:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:03.018 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:02 smithi077 bash[83127]: cluster 2024-09-13T00:41:01.862345+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e392: 8 total, 8 up, 8 in 2024-09-13T00:41:03.019 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:02 smithi077 bash[83127]: cluster 2024-09-13T00:41:02.372312+0000 mgr.y (mgr.25304) 84 : cluster [DBG] pgmap v167: 105 pgs: 105 active+clean; 81 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 4.0 MiB/s wr, 175 op/s 2024-09-13T00:41:03.019 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:02 smithi077 bash[80681]: cluster 2024-09-13T00:41:01.862345+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e392: 8 total, 8 up, 8 in 2024-09-13T00:41:03.019 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:02 smithi077 bash[80681]: cluster 2024-09-13T00:41:02.372312+0000 mgr.y (mgr.25304) 84 : cluster [DBG] pgmap v167: 105 pgs: 105 active+clean; 81 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 4.0 MiB/s wr, 175 op/s 2024-09-13T00:41:03.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:02 smithi088 bash[24895]: cluster 2024-09-13T00:41:01.862345+0000 mon.a (mon.0) 233 : cluster [DBG] osdmap e392: 8 total, 8 up, 8 in 2024-09-13T00:41:03.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:02 smithi088 bash[24895]: cluster 2024-09-13T00:41:02.372312+0000 mgr.y (mgr.25304) 84 : cluster [DBG] pgmap v167: 105 pgs: 105 active+clean; 81 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 4.0 MiB/s wr, 175 op/s 2024-09-13T00:41:03.636 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.CloneFlatten (2275 ms) 2024-09-13T00:41:03.636 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.Stress 2024-09-13T00:41:03.695 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 25063368~1180573 2024-09-13T00:41:03.714 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 8472580~7408442 2024-09-13T00:41:03.716 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 30779515~3926131 2024-09-13T00:41:03.737 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 81516683~745588 2024-09-13T00:41:03.740 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 63897341~7497928 2024-09-13T00:41:03.773 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 78558914~4166797 2024-09-13T00:41:03.775 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 2793181~5999368 2024-09-13T00:41:03.798 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 41998836~6365595 2024-09-13T00:41:03.799 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap0 2024-09-13T00:41:03.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:03 smithi088 bash[61791]: ts=2024-09-13T00:41:03.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:03.919 INFO:tasks.workunit.client.0.smithi077.stdout:clone image226 -> image227 2024-09-13T00:41:04.045 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 147793349 2024-09-13T00:41:04.059 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 30763911~5593418 2024-09-13T00:41:04.094 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 143842863~1129830 2024-09-13T00:41:04.095 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 122351017~5660068 2024-09-13T00:41:04.129 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 116613597~8010365 2024-09-13T00:41:04.132 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 24805859~1394827 2024-09-13T00:41:04.144 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 80402040~5950163 2024-09-13T00:41:04.149 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 103756344~4791444 2024-09-13T00:41:04.170 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 142330465~5462884 2024-09-13T00:41:04.170 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap1 2024-09-13T00:41:04.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:03 smithi077 bash[80681]: cluster 2024-09-13T00:41:02.867698+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e393: 8 total, 8 up, 8 in 2024-09-13T00:41:04.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:03 smithi077 bash[83127]: cluster 2024-09-13T00:41:02.867698+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e393: 8 total, 8 up, 8 in 2024-09-13T00:41:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:03 smithi088 bash[24895]: cluster 2024-09-13T00:41:02.867698+0000 mon.a (mon.0) 234 : cluster [DBG] osdmap e393: 8 total, 8 up, 8 in 2024-09-13T00:41:04.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:04 smithi088 bash[61791]: ts=2024-09-13T00:41:04.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:04.852 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:04 smithi088 bash[61791]: ts=2024-09-13T00:41:04.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:04.922 INFO:tasks.workunit.client.0.smithi077.stdout:clone image227 -> image228 2024-09-13T00:41:05.066 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 194588153 2024-09-13T00:41:05.080 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 60517543~7404442 2024-09-13T00:41:05.130 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 66991158~2409482 2024-09-13T00:41:05.134 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 80743747~4858729 2024-09-13T00:41:05.161 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 29927235~5975813 2024-09-13T00:41:05.181 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 122234181~76039 2024-09-13T00:41:05.198 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 109502212~7892832 2024-09-13T00:41:05.206 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 53277131~1722953 2024-09-13T00:41:05.222 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 51095280~6392882 2024-09-13T00:41:05.225 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap2 2024-09-13T00:41:05.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:04 smithi077 bash[80681]: cluster 2024-09-13T00:41:03.880066+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e394: 8 total, 8 up, 8 in 2024-09-13T00:41:05.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:04 smithi077 bash[80681]: cluster 2024-09-13T00:41:04.373559+0000 mgr.y (mgr.25304) 85 : cluster [DBG] pgmap v170: 105 pgs: 105 active+clean; 261 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s rd, 41 MiB/s wr, 618 op/s 2024-09-13T00:41:05.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:04 smithi077 bash[83127]: cluster 2024-09-13T00:41:03.880066+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e394: 8 total, 8 up, 8 in 2024-09-13T00:41:05.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:04 smithi077 bash[83127]: cluster 2024-09-13T00:41:04.373559+0000 mgr.y (mgr.25304) 85 : cluster [DBG] pgmap v170: 105 pgs: 105 active+clean; 261 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s rd, 41 MiB/s wr, 618 op/s 2024-09-13T00:41:05.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:04 smithi088 bash[24895]: cluster 2024-09-13T00:41:03.880066+0000 mon.a (mon.0) 235 : cluster [DBG] osdmap e394: 8 total, 8 up, 8 in 2024-09-13T00:41:05.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:04 smithi088 bash[24895]: cluster 2024-09-13T00:41:04.373559+0000 mgr.y (mgr.25304) 85 : cluster [DBG] pgmap v170: 105 pgs: 105 active+clean; 261 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s rd, 41 MiB/s wr, 618 op/s 2024-09-13T00:41:05.915 INFO:tasks.workunit.client.0.smithi077.stdout:clone image228 -> image229 2024-09-13T00:41:06.031 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 71535198~5335896 2024-09-13T00:41:06.067 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 134913651~3093429 2024-09-13T00:41:06.073 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 95348254~199958 2024-09-13T00:41:06.080 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 4397496~5349604 2024-09-13T00:41:06.093 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 75683716~6610756 2024-09-13T00:41:06.106 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:05 smithi077 bash[80681]: cluster 2024-09-13T00:41:04.886871+0000 mon.a (mon.0) 236 : cluster [DBG] osdmap e395: 8 total, 8 up, 8 in 2024-09-13T00:41:06.107 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:05 smithi077 bash[83127]: cluster 2024-09-13T00:41:04.886871+0000 mon.a (mon.0) 236 : cluster [DBG] osdmap e395: 8 total, 8 up, 8 in 2024-09-13T00:41:06.124 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 69806911~7432896 2024-09-13T00:41:06.139 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 176008823~7457418 2024-09-13T00:41:06.171 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 181077162~2707403 2024-09-13T00:41:06.173 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap3 2024-09-13T00:41:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:05 smithi088 bash[24895]: cluster 2024-09-13T00:41:04.886871+0000 mon.a (mon.0) 236 : cluster [DBG] osdmap e395: 8 total, 8 up, 8 in 2024-09-13T00:41:06.459 INFO:teuthology.orchestra.run.smithi077.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T00:41:06.459 INFO:teuthology.orchestra.run.smithi077.stdout:alertmanager.a smithi077 *:9093,9094 running (3m) 13s ago 11m 13.8M - 0.25.0 c8568f914cd2 49a4b5607300 2024-09-13T00:41:06.459 INFO:teuthology.orchestra.run.smithi077.stdout:grafana.a smithi088 *:3000 running (3m) 44s ago 10m 75.1M - 9.4.7 954c08fa6188 c369fdf58d31 2024-09-13T00:41:06.459 INFO:teuthology.orchestra.run.smithi077.stdout:mds.foo.smithi077.bigbbf smithi077 running (8m) 13s ago 8m 13.7M - 17.2.7-1627-gbbdadfa3 214985049bdb 30b25ab8f2dd 2024-09-13T00:41:06.459 INFO:teuthology.orchestra.run.smithi077.stdout:mds.foo.smithi088.krmowh smithi088 running (8m) 44s ago 8m 16.7M - 17.2.7-1627-gbbdadfa3 214985049bdb ebef6ef73335 2024-09-13T00:41:06.459 INFO:teuthology.orchestra.run.smithi077.stdout:mgr.x smithi088 *:8443,9283,8765 running (2m) 44s ago 19m 424M - 18.2.4-837-g7a853b17 f21ba1eea74e fdfc79d860d1 2024-09-13T00:41:06.459 INFO:teuthology.orchestra.run.smithi077.stdout:mgr.y smithi077 *:8443,9283,8765 running (3m) 13s ago 21m 474M - 18.2.4-837-g7a853b17 f21ba1eea74e d46e77b45675 2024-09-13T00:41:06.459 INFO:teuthology.orchestra.run.smithi077.stdout:mon.a smithi077 running (2m) 13s ago 22m 47.1M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 3edacf970db4 2024-09-13T00:41:06.460 INFO:teuthology.orchestra.run.smithi077.stdout:mon.b smithi088 running (20m) 44s ago 20m 52.6M 2048M 17.2.7-1627-gbbdadfa3 214985049bdb f980dbb0ec7c 2024-09-13T00:41:06.460 INFO:teuthology.orchestra.run.smithi077.stdout:mon.c smithi077 running (26s) 13s ago 20m 19.4M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 0a377e5bd304 2024-09-13T00:41:06.460 INFO:teuthology.orchestra.run.smithi077.stdout:node-exporter.a smithi077 *:9100 running (3m) 13s ago 11m 8051k - 1.5.0 0da6a335fe13 2aa0a10501f7 2024-09-13T00:41:06.460 INFO:teuthology.orchestra.run.smithi077.stdout:node-exporter.b smithi088 *:9100 running (3m) 44s ago 11m 8051k - 1.5.0 0da6a335fe13 54d4c3d89eab 2024-09-13T00:41:06.460 INFO:teuthology.orchestra.run.smithi077.stdout:osd.0 smithi077 running (18m) 13s ago 18m 115M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb a29e831be9aa 2024-09-13T00:41:06.460 INFO:teuthology.orchestra.run.smithi077.stdout:osd.1 smithi077 running (17m) 13s ago 17m 108M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 8590b3b4d2e9 2024-09-13T00:41:06.460 INFO:teuthology.orchestra.run.smithi077.stdout:osd.2 smithi077 running (16m) 13s ago 17m 135M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb b2ecae227e58 2024-09-13T00:41:06.460 INFO:teuthology.orchestra.run.smithi077.stdout:osd.3 smithi077 running (16m) 13s ago 16m 117M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb d8c3c25f4ebd 2024-09-13T00:41:06.460 INFO:teuthology.orchestra.run.smithi077.stdout:osd.4 smithi088 running (15m) 44s ago 15m 97.5M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 2c9ea6919060 2024-09-13T00:41:06.460 INFO:teuthology.orchestra.run.smithi077.stdout:osd.5 smithi088 running (14m) 44s ago 14m 100M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb e89a6b298b28 2024-09-13T00:41:06.460 INFO:teuthology.orchestra.run.smithi077.stdout:osd.6 smithi088 running (13m) 44s ago 13m 124M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 599c9665873c 2024-09-13T00:41:06.460 INFO:teuthology.orchestra.run.smithi077.stdout:osd.7 smithi088 running (12m) 44s ago 12m 134M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 0ff0388ffa3c 2024-09-13T00:41:06.461 INFO:teuthology.orchestra.run.smithi077.stdout:prometheus.a smithi088 *:9095 running (2m) 44s ago 11m 51.8M - 2.43.0 a07b618ecd1d 5fb440283869 2024-09-13T00:41:06.945 INFO:tasks.workunit.client.0.smithi077.stdout:clone image229 -> image230 2024-09-13T00:41:07.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:06 smithi077 bash[80681]: cluster 2024-09-13T00:41:05.890568+0000 mon.a (mon.0) 237 : cluster [DBG] osdmap e396: 8 total, 8 up, 8 in 2024-09-13T00:41:07.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:06 smithi077 bash[80681]: cluster 2024-09-13T00:41:06.373977+0000 mgr.y (mgr.25304) 86 : cluster [DBG] pgmap v173: 105 pgs: 105 active+clean; 261 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s rd, 41 MiB/s wr, 618 op/s 2024-09-13T00:41:07.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:06 smithi077 bash[83127]: cluster 2024-09-13T00:41:05.890568+0000 mon.a (mon.0) 237 : cluster [DBG] osdmap e396: 8 total, 8 up, 8 in 2024-09-13T00:41:07.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:06 smithi077 bash[83127]: cluster 2024-09-13T00:41:06.373977+0000 mgr.y (mgr.25304) 86 : cluster [DBG] pgmap v173: 105 pgs: 105 active+clean; 261 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s rd, 41 MiB/s wr, 618 op/s 2024-09-13T00:41:07.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:06 smithi088 bash[24895]: cluster 2024-09-13T00:41:05.890568+0000 mon.a (mon.0) 237 : cluster [DBG] osdmap e396: 8 total, 8 up, 8 in 2024-09-13T00:41:08.618 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:06 smithi088 bash[24895]: cluster 2024-09-13T00:41:06.373977+0000 mgr.y (mgr.25304) 86 : cluster [DBG] pgmap v173: 105 pgs: 105 active+clean; 261 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 48 MiB/s rd, 41 MiB/s wr, 618 op/s 2024-09-13T00:41:08.619 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:07 smithi088 bash[24895]: audit 2024-09-13T00:41:06.449476+0000 mgr.y (mgr.25304) 87 : audit [DBG] from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:41:08.619 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:07 smithi088 bash[24895]: cluster 2024-09-13T00:41:06.906255+0000 mon.a (mon.0) 238 : cluster [DBG] osdmap e397: 8 total, 8 up, 8 in 2024-09-13T00:41:08.620 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.Stress (4159 ms) 2024-09-13T00:41:08.620 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.NoSnaps_LargerDstObjSize 2024-09-13T00:41:08.620 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.NoSnaps_LargerDstObjSize (461 ms) 2024-09-13T00:41:08.620 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.Snaps_LargerDstObjSize 2024-09-13T00:41:08.620 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:07 smithi077 bash[80681]: audit 2024-09-13T00:41:06.449476+0000 mgr.y (mgr.25304) 87 : audit [DBG] from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:41:08.620 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:07 smithi077 bash[80681]: cluster 2024-09-13T00:41:06.906255+0000 mon.a (mon.0) 238 : cluster [DBG] osdmap e397: 8 total, 8 up, 8 in 2024-09-13T00:41:08.621 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:07 smithi077 bash[83127]: audit 2024-09-13T00:41:06.449476+0000 mgr.y (mgr.25304) 87 : audit [DBG] from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:41:08.621 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:07 smithi077 bash[83127]: cluster 2024-09-13T00:41:06.906255+0000 mon.a (mon.0) 238 : cluster [DBG] osdmap e397: 8 total, 8 up, 8 in 2024-09-13T00:41:08.622 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph orch upgrade pause' 2024-09-13T00:41:08.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:41:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:41:08] "GET /metrics HTTP/1.1" 200 36438 "" "Prometheus/2.43.0" 2024-09-13T00:41:09.101 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:08 smithi088 bash[61791]: ts=2024-09-13T00:41:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:09.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[80681]: cluster 2024-09-13T00:41:08.374358+0000 mgr.y (mgr.25304) 88 : cluster [DBG] pgmap v175: 105 pgs: 105 active+clean; 261 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 42 MiB/s rd, 36 MiB/s wr, 550 op/s 2024-09-13T00:41:09.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[80681]: audit 2024-09-13T00:41:08.375497+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:41:09.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[80681]: audit 2024-09-13T00:41:08.389503+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:41:09.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[80681]: audit 2024-09-13T00:41:08.392373+0000 mon.b (mon.1) 146 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:09.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[80681]: audit 2024-09-13T00:41:08.394177+0000 mon.b (mon.1) 147 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:41:09.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[80681]: audit 2024-09-13T00:41:08.406262+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:41:09.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[80681]: audit 2024-09-13T00:41:08.558723+0000 mon.b (mon.1) 148 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:41:09.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[80681]: audit 2024-09-13T00:41:08.565818+0000 mon.b (mon.1) 149 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:41:09.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[80681]: audit 2024-09-13T00:41:08.568796+0000 mon.b (mon.1) 150 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T00:41:09.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[80681]: audit 2024-09-13T00:41:08.570757+0000 mon.b (mon.1) 151 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-13T00:41:09.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[80681]: cluster 2024-09-13T00:41:08.912195+0000 mon.a (mon.0) 242 : cluster [DBG] osdmap e398: 8 total, 8 up, 8 in 2024-09-13T00:41:09.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[83127]: cluster 2024-09-13T00:41:08.374358+0000 mgr.y (mgr.25304) 88 : cluster [DBG] pgmap v175: 105 pgs: 105 active+clean; 261 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 42 MiB/s rd, 36 MiB/s wr, 550 op/s 2024-09-13T00:41:09.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[83127]: audit 2024-09-13T00:41:08.375497+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:41:09.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[83127]: audit 2024-09-13T00:41:08.389503+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:41:09.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[83127]: audit 2024-09-13T00:41:08.392373+0000 mon.b (mon.1) 146 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:09.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[83127]: audit 2024-09-13T00:41:08.394177+0000 mon.b (mon.1) 147 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:41:09.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[83127]: audit 2024-09-13T00:41:08.406262+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:41:09.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[83127]: audit 2024-09-13T00:41:08.558723+0000 mon.b (mon.1) 148 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:41:09.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[83127]: audit 2024-09-13T00:41:08.565818+0000 mon.b (mon.1) 149 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:41:09.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[83127]: audit 2024-09-13T00:41:08.568796+0000 mon.b (mon.1) 150 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T00:41:09.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[83127]: audit 2024-09-13T00:41:08.570757+0000 mon.b (mon.1) 151 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-13T00:41:09.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:09 smithi077 bash[83127]: cluster 2024-09-13T00:41:08.912195+0000 mon.a (mon.0) 242 : cluster [DBG] osdmap e398: 8 total, 8 up, 8 in 2024-09-13T00:41:09.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:09 smithi088 bash[24895]: cluster 2024-09-13T00:41:08.374358+0000 mgr.y (mgr.25304) 88 : cluster [DBG] pgmap v175: 105 pgs: 105 active+clean; 261 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 42 MiB/s rd, 36 MiB/s wr, 550 op/s 2024-09-13T00:41:09.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:09 smithi088 bash[24895]: audit 2024-09-13T00:41:08.375497+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:41:09.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:09 smithi088 bash[24895]: audit 2024-09-13T00:41:08.389503+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:41:09.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:09 smithi088 bash[24895]: audit 2024-09-13T00:41:08.392373+0000 mon.b (mon.1) 146 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:09.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:09 smithi088 bash[24895]: audit 2024-09-13T00:41:08.394177+0000 mon.b (mon.1) 147 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:41:09.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:09 smithi088 bash[24895]: audit 2024-09-13T00:41:08.406262+0000 mon.a (mon.0) 241 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:41:09.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:09 smithi088 bash[24895]: audit 2024-09-13T00:41:08.558723+0000 mon.b (mon.1) 148 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:41:09.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:09 smithi088 bash[24895]: audit 2024-09-13T00:41:08.565818+0000 mon.b (mon.1) 149 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:41:09.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:09 smithi088 bash[24895]: audit 2024-09-13T00:41:08.568796+0000 mon.b (mon.1) 150 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-13T00:41:09.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:09 smithi088 bash[24895]: audit 2024-09-13T00:41:08.570757+0000 mon.b (mon.1) 151 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-13T00:41:09.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:09 smithi088 bash[24895]: cluster 2024-09-13T00:41:08.912195+0000 mon.a (mon.0) 242 : cluster [DBG] osdmap e398: 8 total, 8 up, 8 in 2024-09-13T00:41:10.492 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:10 smithi088 bash[24895]: cephadm 2024-09-13T00:41:08.572921+0000 mgr.y (mgr.25304) 89 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-09-13T00:41:10.493 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:10 smithi088 bash[24895]: cluster 2024-09-13T00:41:09.916254+0000 mon.a (mon.0) 243 : cluster [DBG] osdmap e399: 8 total, 8 up, 8 in 2024-09-13T00:41:10.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:10 smithi077 bash[80681]: cephadm 2024-09-13T00:41:08.572921+0000 mgr.y (mgr.25304) 89 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-09-13T00:41:10.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:10 smithi077 bash[80681]: cluster 2024-09-13T00:41:09.916254+0000 mon.a (mon.0) 243 : cluster [DBG] osdmap e399: 8 total, 8 up, 8 in 2024-09-13T00:41:10.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:10 smithi077 bash[83127]: cephadm 2024-09-13T00:41:08.572921+0000 mgr.y (mgr.25304) 89 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-09-13T00:41:10.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:10 smithi077 bash[83127]: cluster 2024-09-13T00:41:09.916254+0000 mon.a (mon.0) 243 : cluster [DBG] osdmap e399: 8 total, 8 up, 8 in 2024-09-13T00:41:11.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:11 smithi077 bash[80681]: cluster 2024-09-13T00:41:10.375021+0000 mgr.y (mgr.25304) 90 : cluster [DBG] pgmap v178: 105 pgs: 105 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s rd, 19 MiB/s wr, 551 op/s 2024-09-13T00:41:11.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:11 smithi077 bash[80681]: cluster 2024-09-13T00:41:10.924244+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e400: 8 total, 8 up, 8 in 2024-09-13T00:41:11.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:11 smithi077 bash[80681]: audit 2024-09-13T00:41:10.936316+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:41:11.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:11 smithi077 bash[80681]: audit 2024-09-13T00:41:10.939696+0000 mon.b (mon.1) 152 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:41:11.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:11 smithi077 bash[80681]: audit 2024-09-13T00:41:10.940941+0000 mon.b (mon.1) 153 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:41:11.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:11 smithi077 bash[80681]: audit 2024-09-13T00:41:10.942148+0000 mon.b (mon.1) 154 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:11.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:11 smithi077 bash[83127]: cluster 2024-09-13T00:41:10.375021+0000 mgr.y (mgr.25304) 90 : cluster [DBG] pgmap v178: 105 pgs: 105 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s rd, 19 MiB/s wr, 551 op/s 2024-09-13T00:41:11.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:11 smithi077 bash[83127]: cluster 2024-09-13T00:41:10.924244+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e400: 8 total, 8 up, 8 in 2024-09-13T00:41:11.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:11 smithi077 bash[83127]: audit 2024-09-13T00:41:10.936316+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:41:11.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:11 smithi077 bash[83127]: audit 2024-09-13T00:41:10.939696+0000 mon.b (mon.1) 152 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:41:11.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:11 smithi077 bash[83127]: audit 2024-09-13T00:41:10.940941+0000 mon.b (mon.1) 153 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:41:11.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:11 smithi077 bash[83127]: audit 2024-09-13T00:41:10.942148+0000 mon.b (mon.1) 154 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:11.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:11 smithi088 bash[24895]: cluster 2024-09-13T00:41:10.375021+0000 mgr.y (mgr.25304) 90 : cluster [DBG] pgmap v178: 105 pgs: 105 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s rd, 19 MiB/s wr, 551 op/s 2024-09-13T00:41:11.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:11 smithi088 bash[24895]: cluster 2024-09-13T00:41:10.924244+0000 mon.a (mon.0) 244 : cluster [DBG] osdmap e400: 8 total, 8 up, 8 in 2024-09-13T00:41:11.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:11 smithi088 bash[24895]: audit 2024-09-13T00:41:10.936316+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.25304 ' entity='mgr.y' 2024-09-13T00:41:11.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:11 smithi088 bash[24895]: audit 2024-09-13T00:41:10.939696+0000 mon.b (mon.1) 152 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:41:11.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:11 smithi088 bash[24895]: audit 2024-09-13T00:41:10.940941+0000 mon.b (mon.1) 153 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:41:11.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:11 smithi088 bash[24895]: audit 2024-09-13T00:41:10.942148+0000 mon.b (mon.1) 154 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:12.425 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.Snaps_LargerDstObjSize (4169 ms) 2024-09-13T00:41:12.425 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.Clone_LargerDstObjSize 2024-09-13T00:41:12.557 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:12 smithi088 bash[24895]: cephadm 2024-09-13T00:41:10.918414+0000 mgr.y (mgr.25304) 91 : cephadm [INF] Upgrade: Updating mon.b 2024-09-13T00:41:12.557 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:12 smithi088 bash[24895]: cephadm 2024-09-13T00:41:10.944506+0000 mgr.y (mgr.25304) 92 : cephadm [INF] Deploying daemon mon.b on smithi088 2024-09-13T00:41:12.557 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:12 smithi088 bash[24895]: cluster 2024-09-13T00:41:11.936176+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e401: 8 total, 8 up, 8 in 2024-09-13T00:41:12.557 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:12 smithi088 bash[61791]: ts=2024-09-13T00:41:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:12.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:12 smithi077 bash[80681]: cephadm 2024-09-13T00:41:10.918414+0000 mgr.y (mgr.25304) 91 : cephadm [INF] Upgrade: Updating mon.b 2024-09-13T00:41:12.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:12 smithi077 bash[80681]: cephadm 2024-09-13T00:41:10.944506+0000 mgr.y (mgr.25304) 92 : cephadm [INF] Deploying daemon mon.b on smithi088 2024-09-13T00:41:12.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:12 smithi077 bash[80681]: cluster 2024-09-13T00:41:11.936176+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e401: 8 total, 8 up, 8 in 2024-09-13T00:41:12.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:12 smithi077 bash[83127]: cephadm 2024-09-13T00:41:10.918414+0000 mgr.y (mgr.25304) 91 : cephadm [INF] Upgrade: Updating mon.b 2024-09-13T00:41:12.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:12 smithi077 bash[83127]: cephadm 2024-09-13T00:41:10.944506+0000 mgr.y (mgr.25304) 92 : cephadm [INF] Deploying daemon mon.b on smithi088 2024-09-13T00:41:12.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:12 smithi077 bash[83127]: cluster 2024-09-13T00:41:11.936176+0000 mon.a (mon.0) 246 : cluster [DBG] osdmap e401: 8 total, 8 up, 8 in 2024-09-13T00:41:13.410 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:41:13.601 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:41:13 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:13.602 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:41:13 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:13.602 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:41:13 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:13.602 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 00:41:13 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:13.603 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:41:13 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:13.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:13 smithi088 bash[24895]: cluster 2024-09-13T00:41:12.375735+0000 mgr.y (mgr.25304) 93 : cluster [DBG] pgmap v181: 105 pgs: 105 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 22 MiB/s wr, 618 op/s 2024-09-13T00:41:13.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:13 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:13.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:13 smithi088 bash[61791]: ts=2024-09-13T00:41:13.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:13.604 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:13 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:13.604 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:41:13 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:13.604 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:41:13 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:13.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:13 smithi077 bash[80681]: cluster 2024-09-13T00:41:12.375735+0000 mgr.y (mgr.25304) 93 : cluster [DBG] pgmap v181: 105 pgs: 105 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 22 MiB/s wr, 618 op/s 2024-09-13T00:41:13.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:13 smithi077 bash[83127]: cluster 2024-09-13T00:41:12.375735+0000 mgr.y (mgr.25304) 93 : cluster [DBG] pgmap v181: 105 pgs: 105 active+clean; 358 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 33 MiB/s rd, 22 MiB/s wr, 618 op/s 2024-09-13T00:41:14.023 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:13 smithi088 systemd[1]: Stopping Ceph mon.b for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:41:14.024 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:13 smithi088 bash[24895]: debug 2024-09-13T00:41:13.761+0000 7f97b70ae640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T00:41:14.024 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:13 smithi088 bash[24895]: debug 2024-09-13T00:41:13.761+0000 7f97b70ae640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-09-13T00:41:14.139 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:41:13 smithi077 bash[73137]: [13/Sep/2024:00:41:13] ENGINE Bus STOPPING 2024-09-13T00:41:14.139 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:41:14 smithi077 bash[73137]: [13/Sep/2024:00:41:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:41:14.139 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:41:14 smithi077 bash[73137]: [13/Sep/2024:00:41:14] ENGINE Bus STOPPED 2024-09-13T00:41:14.139 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:41:14 smithi077 bash[73137]: [13/Sep/2024:00:41:14] ENGINE Bus STARTING 2024-09-13T00:41:14.139 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:41:14 smithi077 bash[73137]: [13/Sep/2024:00:41:14] ENGINE Serving on http://:::9283 2024-09-13T00:41:14.139 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:41:14 smithi077 bash[73137]: [13/Sep/2024:00:41:14] ENGINE Bus STARTED 2024-09-13T00:41:14.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:14 smithi088 bash[68845]: ceph-9218306c-7165-11ef-bceb-c7b262605968-mon-b 2024-09-13T00:41:14.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:14 smithi088 bash[68904]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-mon-b 2024-09-13T00:41:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:14 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-13T00:41:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:14 smithi088 systemd[1]: Stopped Ceph mon.b for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:41:14.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:14 smithi088 bash[61791]: ts=2024-09-13T00:41:14.453Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:14.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:14 smithi088 bash[61791]: ts=2024-09-13T00:41:14.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:15.101 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:41:14 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:15.102 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:41:14 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:15.102 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:41:14 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:15.102 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 00:41:14 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:15.102 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:14 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:15.103 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:41:14 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:15.103 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:41:14 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:15.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:14 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:15.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:14 smithi088 systemd[1]: Started Ceph mon.b for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:41:15.104 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:41:14 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:41:16.112 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.929+0000 7f4ed7f3fb00 0 set uid:gid to 167:167 (ceph:ceph) 2024-09-13T00:41:16.112 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.929+0000 7f4ed7f3fb00 0 ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable), process ceph-mon, pid 7 2024-09-13T00:41:16.112 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.929+0000 7f4ed7f3fb00 0 pidfile_write: ignore empty --pid-file 2024-09-13T00:41:16.112 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 0 load: jerasure load: lrc 2024-09-13T00:41:16.112 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: RocksDB version: 7.9.2 2024-09-13T00:41:16.112 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Git sha 0 2024-09-13T00:41:16.112 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Compile date 2024-09-10 15:58:06 2024-09-13T00:41:16.112 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: DB SUMMARY 2024-09-13T00:41:16.112 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: DB Session ID: H9BRXJQPFOSXPCXZKNMY 2024-09-13T00:41:16.113 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: CURRENT file: CURRENT 2024-09-13T00:41:16.113 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: IDENTITY file: IDENTITY 2024-09-13T00:41:16.113 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: MANIFEST file: MANIFEST-000009 size: 2052 Bytes 2024-09-13T00:41:16.113 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000042.sst 2024-09-13T00:41:16.113 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000040.log size: 734108 ; 2024-09-13T00:41:16.113 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.error_if_exists: 0 2024-09-13T00:41:16.113 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.create_if_missing: 0 2024-09-13T00:41:16.113 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.paranoid_checks: 1 2024-09-13T00:41:16.113 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-09-13T00:41:16.113 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-13T00:41:16.113 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-13T00:41:16.113 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.env: 0x5630c626dbc0 2024-09-13T00:41:16.113 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.fs: PosixFileSystem 2024-09-13T00:41:16.114 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.info_log: 0x5630c8678be0 2024-09-13T00:41:16.114 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_file_opening_threads: 16 2024-09-13T00:41:16.114 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.statistics: (nil) 2024-09-13T00:41:16.114 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.use_fsync: 0 2024-09-13T00:41:16.114 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_log_file_size: 0 2024-09-13T00:41:16.115 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-13T00:41:16.115 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.log_file_time_to_roll: 0 2024-09-13T00:41:16.115 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.keep_log_file_num: 1000 2024-09-13T00:41:16.115 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.recycle_log_file_num: 0 2024-09-13T00:41:16.115 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.allow_fallocate: 1 2024-09-13T00:41:16.115 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.allow_mmap_reads: 0 2024-09-13T00:41:16.115 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.allow_mmap_writes: 0 2024-09-13T00:41:16.115 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.use_direct_reads: 0 2024-09-13T00:41:16.115 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-13T00:41:16.115 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.create_missing_column_families: 0 2024-09-13T00:41:16.115 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.db_log_dir: 2024-09-13T00:41:16.116 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.wal_dir: 2024-09-13T00:41:16.116 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.table_cache_numshardbits: 6 2024-09-13T00:41:16.116 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-09-13T00:41:16.116 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-09-13T00:41:16.116 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-13T00:41:16.116 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-13T00:41:16.116 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-09-13T00:41:16.116 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.advise_random_on_open: 1 2024-09-13T00:41:16.116 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.db_write_buffer_size: 0 2024-09-13T00:41:16.116 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.write_buffer_manager: 0x5630c8686320 2024-09-13T00:41:16.116 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-13T00:41:16.116 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-13T00:41:16.117 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.use_adaptive_mutex: 0 2024-09-13T00:41:16.117 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.rate_limiter: (nil) 2024-09-13T00:41:16.117 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-13T00:41:16.117 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.wal_recovery_mode: 2 2024-09-13T00:41:16.117 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.enable_thread_tracking: 0 2024-09-13T00:41:16.117 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.enable_pipelined_write: 0 2024-09-13T00:41:16.117 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.unordered_write: 0 2024-09-13T00:41:16.117 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-13T00:41:16.117 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-13T00:41:16.117 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-13T00:41:16.117 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-13T00:41:16.117 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.row_cache: None 2024-09-13T00:41:16.117 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.wal_filter: None 2024-09-13T00:41:16.117 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-13T00:41:16.118 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.allow_ingest_behind: 0 2024-09-13T00:41:16.118 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.two_write_queues: 0 2024-09-13T00:41:16.118 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.manual_wal_flush: 0 2024-09-13T00:41:16.118 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.wal_compression: 0 2024-09-13T00:41:16.118 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.atomic_flush: 0 2024-09-13T00:41:16.118 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-13T00:41:16.118 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.persist_stats_to_disk: 0 2024-09-13T00:41:16.118 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-09-13T00:41:16.118 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.log_readahead_size: 0 2024-09-13T00:41:16.118 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-13T00:41:16.118 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.best_efforts_recovery: 0 2024-09-13T00:41:16.118 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-13T00:41:16.118 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-13T00:41:16.119 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.allow_data_in_errors: 0 2024-09-13T00:41:16.119 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.db_host_id: __hostname__ 2024-09-13T00:41:16.119 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.enforce_single_del_contracts: true 2024-09-13T00:41:16.119 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_background_jobs: 2 2024-09-13T00:41:16.119 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_background_compactions: -1 2024-09-13T00:41:16.119 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_subcompactions: 1 2024-09-13T00:41:16.119 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-13T00:41:16.119 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-13T00:41:16.119 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.delayed_write_rate : 16777216 2024-09-13T00:41:16.119 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_total_wal_size: 0 2024-09-13T00:41:16.119 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-13T00:41:16.119 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.stats_dump_period_sec: 600 2024-09-13T00:41:16.119 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.stats_persist_period_sec: 600 2024-09-13T00:41:16.120 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-13T00:41:16.120 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_open_files: -1 2024-09-13T00:41:16.120 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.bytes_per_sync: 0 2024-09-13T00:41:16.120 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-09-13T00:41:16.120 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-09-13T00:41:16.120 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compaction_readahead_size: 0 2024-09-13T00:41:16.120 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_background_flushes: -1 2024-09-13T00:41:16.120 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Compression algorithms supported: 2024-09-13T00:41:16.120 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: kZSTD supported: 0 2024-09-13T00:41:16.120 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: kXpressCompression supported: 0 2024-09-13T00:41:16.120 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: kBZip2Compression supported: 0 2024-09-13T00:41:16.120 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-13T00:41:16.120 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: kLZ4Compression supported: 1 2024-09-13T00:41:16.121 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: kZlibCompression supported: 1 2024-09-13T00:41:16.121 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: kLZ4HCCompression supported: 1 2024-09-13T00:41:16.121 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: kSnappyCompression supported: 1 2024-09-13T00:41:16.121 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-09-13T00:41:16.121 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-09-13T00:41:16.121 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-09-13T00:41:16.121 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-13T00:41:16.121 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-13T00:41:16.121 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.merge_operator: 2024-09-13T00:41:16.121 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compaction_filter: None 2024-09-13T00:41:16.121 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compaction_filter_factory: None 2024-09-13T00:41:16.121 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.sst_partitioner_factory: None 2024-09-13T00:41:16.121 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-09-13T00:41:16.121 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.table_factory: BlockBasedTable 2024-09-13T00:41:16.122 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5630c86787e0) 2024-09-13T00:41:16.122 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: cache_index_and_filter_blocks: 1 2024-09-13T00:41:16.122 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-13T00:41:16.122 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-13T00:41:16.122 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: pin_top_level_index_and_filter: 1 2024-09-13T00:41:16.122 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: index_type: 0 2024-09-13T00:41:16.122 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: data_block_index_type: 0 2024-09-13T00:41:16.122 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: index_shortening: 1 2024-09-13T00:41:16.122 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: data_block_hash_table_util_ratio: 0.750000 2024-09-13T00:41:16.122 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: checksum: 4 2024-09-13T00:41:16.122 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: no_block_cache: 0 2024-09-13T00:41:16.122 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: block_cache: 0x5630c866f1f0 2024-09-13T00:41:16.122 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: block_cache_name: BinnedLRUCache 2024-09-13T00:41:16.123 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: block_cache_options: 2024-09-13T00:41:16.123 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: capacity : 536870912 2024-09-13T00:41:16.123 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: num_shard_bits : 4 2024-09-13T00:41:16.123 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: strict_capacity_limit : 0 2024-09-13T00:41:16.123 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: high_pri_pool_ratio: 0.000 2024-09-13T00:41:16.123 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: block_cache_compressed: (nil) 2024-09-13T00:41:16.123 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: persistent_cache: (nil) 2024-09-13T00:41:16.123 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: block_size: 4096 2024-09-13T00:41:16.123 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: block_size_deviation: 10 2024-09-13T00:41:16.123 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: block_restart_interval: 16 2024-09-13T00:41:16.123 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: index_block_restart_interval: 1 2024-09-13T00:41:16.123 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: metadata_block_size: 4096 2024-09-13T00:41:16.123 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: partition_filters: 0 2024-09-13T00:41:16.124 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: use_delta_encoding: 1 2024-09-13T00:41:16.124 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: filter_policy: bloomfilter 2024-09-13T00:41:16.124 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: whole_key_filtering: 1 2024-09-13T00:41:16.124 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: verify_compression: 0 2024-09-13T00:41:16.124 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: read_amp_bytes_per_bit: 0 2024-09-13T00:41:16.124 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: format_version: 5 2024-09-13T00:41:16.124 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: enable_index_compression: 1 2024-09-13T00:41:16.124 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: block_align: 0 2024-09-13T00:41:16.124 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: max_auto_readahead_size: 262144 2024-09-13T00:41:16.124 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: prepopulate_block_cache: 0 2024-09-13T00:41:16.124 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: initial_auto_readahead_size: 8192 2024-09-13T00:41:16.124 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: num_file_reads_for_auto_readahead: 2 2024-09-13T00:41:16.124 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.write_buffer_size: 33554432 2024-09-13T00:41:16.125 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_write_buffer_number: 2 2024-09-13T00:41:16.126 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compression: NoCompression 2024-09-13T00:41:16.126 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.bottommost_compression: Disabled 2024-09-13T00:41:16.126 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.prefix_extractor: nullptr 2024-09-13T00:41:16.126 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-13T00:41:16.126 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.num_levels: 7 2024-09-13T00:41:16.126 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-13T00:41:16.126 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-13T00:41:16.126 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-13T00:41:16.127 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-13T00:41:16.127 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-13T00:41:16.127 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-13T00:41:16.127 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-13T00:41:16.127 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-13T00:41:16.127 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-13T00:41:16.127 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-13T00:41:16.127 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-13T00:41:16.127 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-13T00:41:16.127 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compression_opts.window_bits: -14 2024-09-13T00:41:16.127 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compression_opts.level: 32767 2024-09-13T00:41:16.127 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compression_opts.strategy: 0 2024-09-13T00:41:16.127 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-13T00:41:16.128 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-13T00:41:16.128 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-13T00:41:16.128 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-13T00:41:16.128 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compression_opts.enabled: false 2024-09-13T00:41:16.128 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-13T00:41:16.128 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-13T00:41:16.128 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-13T00:41:16.128 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-13T00:41:16.128 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.target_file_size_base: 67108864 2024-09-13T00:41:16.128 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.target_file_size_multiplier: 1 2024-09-13T00:41:16.128 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-13T00:41:16.128 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-13T00:41:16.128 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-13T00:41:16.128 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-13T00:41:16.129 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-13T00:41:16.129 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-13T00:41:16.129 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-13T00:41:16.129 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-13T00:41:16.129 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-13T00:41:16.129 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-13T00:41:16.129 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-13T00:41:16.129 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-13T00:41:16.129 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-13T00:41:16.129 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.arena_block_size: 1048576 2024-09-13T00:41:16.129 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-13T00:41:16.129 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-13T00:41:16.129 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.disable_auto_compactions: 0 2024-09-13T00:41:16.129 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-13T00:41:16.130 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-13T00:41:16.130 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-13T00:41:16.130 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-13T00:41:16.130 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-13T00:41:16.130 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-13T00:41:16.130 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-13T00:41:16.130 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-13T00:41:16.130 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-13T00:41:16.130 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-13T00:41:16.130 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-13T00:41:16.130 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.inplace_update_support: 0 2024-09-13T00:41:16.130 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-09-13T00:41:16.130 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-13T00:41:16.131 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-13T00:41:16.131 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.memtable_huge_page_size: 0 2024-09-13T00:41:16.131 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.bloom_locality: 0 2024-09-13T00:41:16.131 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.max_successive_merges: 0 2024-09-13T00:41:16.131 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-09-13T00:41:16.131 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.paranoid_file_checks: 0 2024-09-13T00:41:16.131 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.force_consistency_checks: 1 2024-09-13T00:41:16.131 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.report_bg_io_stats: 0 2024-09-13T00:41:16.131 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.ttl: 2592000 2024-09-13T00:41:16.131 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-09-13T00:41:16.131 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-13T00:41:16.131 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-13T00:41:16.131 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.enable_blob_files: false 2024-09-13T00:41:16.132 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.min_blob_size: 0 2024-09-13T00:41:16.132 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.blob_file_size: 268435456 2024-09-13T00:41:16.132 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.blob_compression_type: NoCompression 2024-09-13T00:41:16.132 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-09-13T00:41:16.132 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-13T00:41:16.132 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-13T00:41:16.132 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-13T00:41:16.132 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.blob_file_starting_level: 0 2024-09-13T00:41:16.132 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-13T00:41:16.132 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed4d18640 3 rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 42.sst 2024-09-13T00:41:16.132 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T00:41:16.132 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 44, last_sequence is 24361, log_number is 40,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-13T00:41:16.133 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 40 2024-09-13T00:41:16.133 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d59b9ce4-ada9-4e34-9af2-725521cf0dbe 2024-09-13T00:41:16.133 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726188075941176, "job": 1, "event": "recovery_started", "wal_files": [40]} 2024-09-13T00:41:16.133 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.933+0000 7f4ed7f3fb00 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #40 mode 2 2024-09-13T00:41:16.133 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.937+0000 7f4ed7f3fb00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726188075944462, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 45, "file_size": 472938, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 24367, "largest_seqno": 24749, "table_properties": {"data_size": 470118, "index_size": 1332, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 453, "raw_key_size": 3616, "raw_average_key_size": 23, "raw_value_size": 466270, "raw_average_value_size": 2969, "num_data_blocks": 58, "num_entries": 157, "num_filter_entries": 157, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726188075, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d59b9ce4-ada9-4e34-9af2-725521cf0dbe", "db_session_id": "H9BRXJQPFOSXPCXZKNMY", "orig_file_number": 45, "seqno_to_time_mapping": "N/A"}} 2024-09-13T00:41:16.133 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.937+0000 7f4ed7f3fb00 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1726188075944605, "job": 1, "event": "recovery_finished"} 2024-09-13T00:41:16.133 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.937+0000 7f4ed7f3fb00 4 rocksdb: [db/version_set.cc:5047] Creating manifest 47 2024-09-13T00:41:16.134 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.937+0000 7f4ed7f3fb00 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-13T00:41:16.134 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.941+0000 7f4ed7f3fb00 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000040.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-13T00:41:16.134 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.941+0000 7f4ed7f3fb00 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5630c8698e00 2024-09-13T00:41:16.134 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.941+0000 7f4ed7f3fb00 4 rocksdb: DB pointer 0x5630c8790000 2024-09-13T00:41:16.134 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.941+0000 7f4ecdd0a640 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-13T00:41:16.134 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.941+0000 7f4ecdd0a640 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-13T00:41:16.134 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: ** DB Stats ** 2024-09-13T00:41:16.134 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:41:16.134 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-13T00:41:16.134 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T00:41:16.135 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T00:41:16.135 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-13T00:41:16.135 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-13T00:41:16.135 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-13T00:41:16.135 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: ** Compaction Stats [default] ** 2024-09-13T00:41:16.135 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T00:41:16.135 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-13T00:41:16.135 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: L0 1/0 461.85 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 253.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-09-13T00:41:16.135 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: L6 1/0 12.97 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-13T00:41:16.135 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Sum 2/0 13.42 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 253.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-09-13T00:41:16.135 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 253.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-09-13T00:41:16.135 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: ** Compaction Stats [default] ** 2024-09-13T00:41:16.136 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-13T00:41:16.136 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-13T00:41:16.136 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 253.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-09-13T00:41:16.136 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-13T00:41:16.136 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Uptime(secs): 0.0 total, 0.0 interval 2024-09-13T00:41:16.136 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Flush(GB): cumulative 0.000, interval 0.000 2024-09-13T00:41:16.136 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-13T00:41:16.136 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: AddFile(Total Files): cumulative 0, interval 0 2024-09-13T00:41:16.136 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: AddFile(L0 Files): cumulative 0, interval 0 2024-09-13T00:41:16.137 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: AddFile(Keys): cumulative 0, interval 0 2024-09-13T00:41:16.137 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Cumulative compaction: 0.00 GB write, 54.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:41:16.137 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Interval compaction: 0.00 GB write, 54.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-13T00:41:16.137 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-13T00:41:16.137 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Block cache BinnedLRUCache@0x5630c866f1f0#7 capacity: 512.00 MB usage: 1.95 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-09-13T00:41:16.137 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Block cache entry stats(count,size,portion): FilterBlock(1,0.50 KB,9.53674e-05%) IndexBlock(1,1.45 KB,0.000277162%) Misc(1,0.00 KB,0%) 2024-09-13T00:41:16.137 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: ** File Read Latency Histogram By Level [default] ** 2024-09-13T00:41:16.137 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.941+0000 7f4ed7f3fb00 0 starting mon.b rank 1 at public addrs [v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0] at bind addrs [v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:41:16.137 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.945+0000 7f4ed7f3fb00 1 mon.b@-1(???) e3 preinit fsid 9218306c-7165-11ef-bceb-c7b262605968 2024-09-13T00:41:16.137 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.945+0000 7f4ed7f3fb00 0 mon.b@-1(???).mds e9 new map 2024-09-13T00:41:16.137 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.945+0000 7f4ed7f3fb00 0 mon.b@-1(???).mds e9 print_map 2024-09-13T00:41:16.137 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: e9 2024-09-13T00:41:16.138 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: enable_multiple, ever_enabled_multiple: 1,1 2024-09-13T00:41:16.138 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T00:41:16.138 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: legacy client fscid: 1 2024-09-13T00:41:16.138 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: 2024-09-13T00:41:16.138 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Filesystem 'foo' (1) 2024-09-13T00:41:16.138 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: fs_name foo 2024-09-13T00:41:16.138 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: epoch 9 2024-09-13T00:41:16.138 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-13T00:41:16.138 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: created 2024-09-13T00:32:23.134111+0000 2024-09-13T00:41:16.138 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: modified 2024-09-13T00:32:33.870108+0000 2024-09-13T00:41:16.138 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: tableserver 0 2024-09-13T00:41:16.139 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: root 0 2024-09-13T00:41:16.139 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: session_timeout 60 2024-09-13T00:41:16.139 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: session_autoclose 300 2024-09-13T00:41:16.139 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: max_file_size 1099511627776 2024-09-13T00:41:16.139 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: max_xattr_size 65536 2024-09-13T00:41:16.139 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: required_client_features {} 2024-09-13T00:41:16.139 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: last_failure 0 2024-09-13T00:41:16.139 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: last_failure_osd_epoch 63 2024-09-13T00:41:16.139 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-13T00:41:16.139 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: max_mds 1 2024-09-13T00:41:16.139 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: in 0 2024-09-13T00:41:16.140 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: up {0=24349} 2024-09-13T00:41:16.140 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: failed 2024-09-13T00:41:16.140 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: damaged 2024-09-13T00:41:16.140 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: stopped 2024-09-13T00:41:16.140 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: data_pools [4] 2024-09-13T00:41:16.140 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: metadata_pool 3 2024-09-13T00:41:16.140 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: inline_data disabled 2024-09-13T00:41:16.140 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: balancer 2024-09-13T00:41:16.140 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: bal_rank_mask -1 2024-09-13T00:41:16.140 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: standby_count_wanted 1 2024-09-13T00:41:16.140 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: [mds.foo.smithi088.krmowh{0:24349} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.88:6832/1144032959,v1:172.21.15.88:6833/1144032959] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T00:41:16.140 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: 2024-09-13T00:41:16.141 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: 2024-09-13T00:41:16.141 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: Standby daemons: 2024-09-13T00:41:16.141 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: 2024-09-13T00:41:16.141 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: [mds.foo.smithi077.bigbbf{-1:14604} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.77:6834/3765041927,v1:172.21.15.77:6835/3765041927] compat {c=[1],r=[1],i=[7ff]}] 2024-09-13T00:41:16.141 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.945+0000 7f4ed7f3fb00 0 mon.b@-1(???).osd e402 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-13T00:41:16.141 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.945+0000 7f4ed7f3fb00 0 mon.b@-1(???).osd e402 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T00:41:16.141 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.945+0000 7f4ed7f3fb00 0 mon.b@-1(???).osd e402 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T00:41:16.141 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.945+0000 7f4ed7f3fb00 0 mon.b@-1(???).osd e402 crush map has features 432629239337189376, adjusting msgr requires 2024-09-13T00:41:16.141 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:15 smithi088 bash[68960]: debug 2024-09-13T00:41:15.945+0000 7f4ed7f3fb00 1 mon.b@-1(???).paxosservice(auth 1..24) refresh upgraded, format 0 -> 3 2024-09-13T00:41:17.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:17 smithi077 bash[80681]: debug 2024-09-13T00:41:17.458+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-09-13T00:41:17.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:17 smithi077 bash[80681]: debug 2024-09-13T00:41:17.458+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:41:17.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:17 smithi077 bash[80681]: debug 2024-09-13T00:41:17.458+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:17.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:17 smithi077 bash[80681]: debug 2024-09-13T00:41:17.458+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:17.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:17 smithi077 bash[80681]: debug 2024-09-13T00:41:17.458+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:17.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:17 smithi077 bash[80681]: debug 2024-09-13T00:41:17.458+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:17.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:17 smithi077 bash[80681]: debug 2024-09-13T00:41:17.458+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:17.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:17 smithi077 bash[80681]: debug 2024-09-13T00:41:17.458+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:17.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:17 smithi077 bash[80681]: debug 2024-09-13T00:41:17.458+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:17.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:17 smithi077 bash[80681]: debug 2024-09-13T00:41:17.458+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:19.028 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:41:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:41:18] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.43.0" 2024-09-13T00:41:19.101 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:18 smithi088 bash[61791]: ts=2024-09-13T00:41:18.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:21.278 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:41:20 smithi077 bash[73137]: debug 2024-09-13T00:41:20.958+0000 7ffa94da1640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-13T00:41:22.524 INFO:teuthology.orchestra.run.smithi077.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:41:22.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[61791]: ts=2024-09-13T00:41:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: audit 2024-09-13T00:41:17.426545+0000 mgr.y (mgr.25304) 96 : audit [DBG] from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:41:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cephadm 2024-09-13T00:41:17.427198+0000 mgr.y (mgr.25304) 97 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:41:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:17.461515+0000 mon.a (mon.0) 261 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:17.461546+0000 mon.a (mon.0) 262 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:17.461574+0000 mon.a (mon.0) 263 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:17.461591+0000 mon.a (mon.0) 264 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:17.461612+0000 mon.a (mon.0) 265 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:17.461633+0000 mon.a (mon.0) 266 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:17.471640+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-13T00:41:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:17.472089+0000 mon.a (mon.0) 267 : cluster [INF] mon.a calling monitor election 2024-09-13T00:41:22.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: audit 2024-09-13T00:41:17.473877+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:41:22.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: audit 2024-09-13T00:41:17.474302+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:41:22.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: audit 2024-09-13T00:41:17.474597+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:41:22.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:18.378308+0000 mgr.y (mgr.25304) 98 : cluster [DBG] pgmap v185: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 4.8 MiB/s wr, 93 op/s 2024-09-13T00:41:22.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:20.379596+0000 mgr.y (mgr.25304) 99 : cluster [DBG] pgmap v186: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 4.3 MiB/s wr, 82 op/s 2024-09-13T00:41:22.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.380271+0000 mgr.y (mgr.25304) 100 : cluster [DBG] pgmap v187: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 3.6 MiB/s wr, 69 op/s 2024-09-13T00:41:22.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.473761+0000 mon.a (mon.0) 271 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T00:41:22.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.482614+0000 mon.a (mon.0) 272 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:41:22.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.503496+0000 mon.a (mon.0) 273 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:41:22.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.503548+0000 mon.a (mon.0) 274 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2024-09-13T00:41:22.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.504041+0000 mon.a (mon.0) 275 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-09-13T00:41:22.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.504350+0000 mon.a (mon.0) 276 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-13T00:41:22.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.514178+0000 mon.a (mon.0) 277 : cluster [ERR] Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-09-13T00:41:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.514207+0000 mon.a (mon.0) 278 : cluster [ERR] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-13T00:41:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.514226+0000 mon.a (mon.0) 279 : cluster [ERR] mon.c (rank 2) addr [v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0] is down (out of quorum) 2024-09-13T00:41:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.514241+0000 mon.a (mon.0) 280 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:41:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.514270+0000 mon.a (mon.0) 281 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.514302+0000 mon.a (mon.0) 282 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.514325+0000 mon.a (mon.0) 283 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.514344+0000 mon.a (mon.0) 284 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.514372+0000 mon.a (mon.0) 285 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.514400+0000 mon.a (mon.0) 286 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.514429+0000 mon.a (mon.0) 287 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: cluster 2024-09-13T00:41:22.514506+0000 mon.a (mon.0) 288 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: audit 2024-09-13T00:41:22.520812+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:22.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: audit 2024-09-13T00:41:22.526758+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:22.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: audit 2024-09-13T00:41:22.535121+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:22.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:22 smithi088 bash[68960]: audit 2024-09-13T00:41:22.537573+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:41:22.762 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: debug 2024-09-13T00:41:22.510+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-09-13T00:41:22.763 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: debug 2024-09-13T00:41:22.510+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-13T00:41:22.763 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: debug 2024-09-13T00:41:22.510+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : mon.c (rank 2) addr [v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0] is down (out of quorum) 2024-09-13T00:41:22.763 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: debug 2024-09-13T00:41:22.510+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:41:22.763 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: debug 2024-09-13T00:41:22.510+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.763 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: debug 2024-09-13T00:41:22.510+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.763 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: debug 2024-09-13T00:41:22.510+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.763 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: debug 2024-09-13T00:41:22.510+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.763 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: debug 2024-09-13T00:41:22.510+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.763 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: debug 2024-09-13T00:41:22.510+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.763 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: debug 2024-09-13T00:41:22.510+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.763 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: debug 2024-09-13T00:41:22.510+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.764 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: audit 2024-09-13T00:41:17.426545+0000 mgr.y (mgr.25304) 96 : audit [DBG] from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:41:22.764 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cephadm 2024-09-13T00:41:17.427198+0000 mgr.y (mgr.25304) 97 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:41:22.764 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:17.461515+0000 mon.a (mon.0) 261 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.764 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:17.461546+0000 mon.a (mon.0) 262 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.764 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:17.461574+0000 mon.a (mon.0) 263 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.764 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:17.461591+0000 mon.a (mon.0) 264 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.764 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:17.461612+0000 mon.a (mon.0) 265 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.765 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:17.461633+0000 mon.a (mon.0) 266 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.765 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:17.471640+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-13T00:41:22.765 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:17.472089+0000 mon.a (mon.0) 267 : cluster [INF] mon.a calling monitor election 2024-09-13T00:41:22.765 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: audit 2024-09-13T00:41:17.473877+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:41:22.765 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: audit 2024-09-13T00:41:17.474302+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:41:22.765 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: audit 2024-09-13T00:41:17.474597+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:41:22.765 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:18.378308+0000 mgr.y (mgr.25304) 98 : cluster [DBG] pgmap v185: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 4.8 MiB/s wr, 93 op/s 2024-09-13T00:41:22.765 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:20.379596+0000 mgr.y (mgr.25304) 99 : cluster [DBG] pgmap v186: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 4.3 MiB/s wr, 82 op/s 2024-09-13T00:41:22.765 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.380271+0000 mgr.y (mgr.25304) 100 : cluster [DBG] pgmap v187: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 3.6 MiB/s wr, 69 op/s 2024-09-13T00:41:22.765 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.473761+0000 mon.a (mon.0) 271 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T00:41:22.766 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.482614+0000 mon.a (mon.0) 272 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:41:22.766 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.503496+0000 mon.a (mon.0) 273 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:41:22.766 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.503548+0000 mon.a (mon.0) 274 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2024-09-13T00:41:22.766 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.504041+0000 mon.a (mon.0) 275 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-09-13T00:41:22.766 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.504350+0000 mon.a (mon.0) 276 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-13T00:41:22.766 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.514178+0000 mon.a (mon.0) 277 : cluster [ERR] Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-09-13T00:41:22.766 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.514207+0000 mon.a (mon.0) 278 : cluster [ERR] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-13T00:41:22.766 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.514226+0000 mon.a (mon.0) 279 : cluster [ERR] mon.c (rank 2) addr [v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0] is down (out of quorum) 2024-09-13T00:41:22.766 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.514241+0000 mon.a (mon.0) 280 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:41:22.766 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.514270+0000 mon.a (mon.0) 281 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.766 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.514302+0000 mon.a (mon.0) 282 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.766 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.514325+0000 mon.a (mon.0) 283 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.767 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.514344+0000 mon.a (mon.0) 284 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.767 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.514372+0000 mon.a (mon.0) 285 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.767 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.514400+0000 mon.a (mon.0) 286 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.767 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.514429+0000 mon.a (mon.0) 287 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.767 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: cluster 2024-09-13T00:41:22.514506+0000 mon.a (mon.0) 288 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:22.767 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: audit 2024-09-13T00:41:22.520812+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:22.767 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: audit 2024-09-13T00:41:22.526758+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:22.767 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: audit 2024-09-13T00:41:22.535121+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:22.767 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:22 smithi077 bash[80681]: audit 2024-09-13T00:41:22.537573+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:41:23.349 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'sleep 60' 2024-09-13T00:41:23.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:23 smithi088 bash[61791]: ts=2024-09-13T00:41:23.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:24.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:24 smithi077 bash[80681]: cluster 2024-09-13T00:41:24.381515+0000 mgr.y (mgr.25304) 101 : cluster [DBG] pgmap v188: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 MiB/s rd, 3.3 MiB/s wr, 63 op/s 2024-09-13T00:41:24.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:24 smithi088 bash[68960]: cluster 2024-09-13T00:41:24.381515+0000 mgr.y (mgr.25304) 101 : cluster [DBG] pgmap v188: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 MiB/s rd, 3.3 MiB/s wr, 63 op/s 2024-09-13T00:41:24.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:24 smithi088 bash[61791]: ts=2024-09-13T00:41:24.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:24.852 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:24 smithi088 bash[61791]: ts=2024-09-13T00:41:24.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:25.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: debug 2024-09-13T00:41:25.061+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-09-13T00:41:25.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: debug 2024-09-13T00:41:25.061+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:41:25.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: debug 2024-09-13T00:41:25.061+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: debug 2024-09-13T00:41:25.061+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: debug 2024-09-13T00:41:25.061+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: debug 2024-09-13T00:41:25.061+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: debug 2024-09-13T00:41:25.061+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: debug 2024-09-13T00:41:25.061+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: debug 2024-09-13T00:41:25.061+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: debug 2024-09-13T00:41:25.061+0000 7fc7975b6640 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: audit 2024-09-13T00:41:17.426545+0000 mgr.y (mgr.25304) 96 : audit [DBG] from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:41:25.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cephadm 2024-09-13T00:41:17.427198+0000 mgr.y (mgr.25304) 97 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:41:25.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:17.461515+0000 mon.a (mon.0) 261 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:17.461546+0000 mon.a (mon.0) 262 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:17.461574+0000 mon.a (mon.0) 263 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:17.461591+0000 mon.a (mon.0) 264 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:17.461612+0000 mon.a (mon.0) 265 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:17.461633+0000 mon.a (mon.0) 266 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:17.471640+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-09-13T00:41:25.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:17.472089+0000 mon.a (mon.0) 267 : cluster [INF] mon.a calling monitor election 2024-09-13T00:41:25.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: audit 2024-09-13T00:41:17.473877+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-13T00:41:25.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: audit 2024-09-13T00:41:17.474302+0000 mon.a (mon.0) 269 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-13T00:41:25.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: audit 2024-09-13T00:41:17.474597+0000 mon.a (mon.0) 270 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-13T00:41:25.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:18.378308+0000 mgr.y (mgr.25304) 98 : cluster [DBG] pgmap v185: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 13 MiB/s rd, 4.8 MiB/s wr, 93 op/s 2024-09-13T00:41:25.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:20.379596+0000 mgr.y (mgr.25304) 99 : cluster [DBG] pgmap v186: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 12 MiB/s rd, 4.3 MiB/s wr, 82 op/s 2024-09-13T00:41:25.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.380271+0000 mgr.y (mgr.25304) 100 : cluster [DBG] pgmap v187: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 3.6 MiB/s wr, 69 op/s 2024-09-13T00:41:25.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.473761+0000 mon.a (mon.0) 271 : cluster [INF] mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-13T00:41:25.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.482614+0000 mon.a (mon.0) 272 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:41:25.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.503496+0000 mon.a (mon.0) 273 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:41:25.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.503548+0000 mon.a (mon.0) 274 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2024-09-13T00:41:25.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.504041+0000 mon.a (mon.0) 275 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-09-13T00:41:25.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.504350+0000 mon.a (mon.0) 276 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-09-13T00:41:25.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.514178+0000 mon.a (mon.0) 277 : cluster [ERR] Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable 2024-09-13T00:41:25.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.514207+0000 mon.a (mon.0) 278 : cluster [ERR] [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-09-13T00:41:25.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.514226+0000 mon.a (mon.0) 279 : cluster [ERR] mon.c (rank 2) addr [v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0] is down (out of quorum) 2024-09-13T00:41:25.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.514241+0000 mon.a (mon.0) 280 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:41:25.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.514270+0000 mon.a (mon.0) 281 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.514302+0000 mon.a (mon.0) 282 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.514325+0000 mon.a (mon.0) 283 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.514344+0000 mon.a (mon.0) 284 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.514372+0000 mon.a (mon.0) 285 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.514400+0000 mon.a (mon.0) 286 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.514429+0000 mon.a (mon.0) 287 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:22.514506+0000 mon.a (mon.0) 288 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: audit 2024-09-13T00:41:22.520812+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:25.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: audit 2024-09-13T00:41:22.526758+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:25.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: audit 2024-09-13T00:41:22.535121+0000 mon.a (mon.0) 291 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:25.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: audit 2024-09-13T00:41:22.537573+0000 mon.a (mon.0) 292 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:41:25.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:24.381515+0000 mgr.y (mgr.25304) 101 : cluster [DBG] pgmap v188: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 9.1 MiB/s rd, 3.3 MiB/s wr, 63 op/s 2024-09-13T00:41:25.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:17.473526+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2024-09-13T00:41:25.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.000798+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2024-09-13T00:41:25.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.001042+0000 mon.a (mon.0) 293 : cluster [INF] mon.a calling monitor election 2024-09-13T00:41:25.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.024102+0000 mon.a (mon.0) 294 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T00:41:25.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.034277+0000 mon.a (mon.0) 295 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:41:25.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.055466+0000 mon.a (mon.0) 296 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:41:25.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.055537+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2024-09-13T00:41:25.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.056204+0000 mon.a (mon.0) 298 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-09-13T00:41:25.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.056507+0000 mon.a (mon.0) 299 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T00:41:25.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.063509+0000 mon.a (mon.0) 300 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-09-13T00:41:25.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.063547+0000 mon.a (mon.0) 301 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:41:25.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.063586+0000 mon.a (mon.0) 302 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.063617+0000 mon.a (mon.0) 303 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.063650+0000 mon.a (mon.0) 304 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.063686+0000 mon.a (mon.0) 305 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.063729+0000 mon.a (mon.0) 306 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.063763+0000 mon.a (mon.0) 307 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.063796+0000 mon.a (mon.0) 308 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.063838+0000 mon.a (mon.0) 309 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:17.473526+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2024-09-13T00:41:25.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.000798+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2024-09-13T00:41:25.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.001042+0000 mon.a (mon.0) 293 : cluster [INF] mon.a calling monitor election 2024-09-13T00:41:25.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.024102+0000 mon.a (mon.0) 294 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T00:41:25.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.034277+0000 mon.a (mon.0) 295 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:41:25.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.055466+0000 mon.a (mon.0) 296 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:41:25.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.055537+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2024-09-13T00:41:25.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.056204+0000 mon.a (mon.0) 298 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-09-13T00:41:25.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.056507+0000 mon.a (mon.0) 299 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T00:41:25.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.063509+0000 mon.a (mon.0) 300 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-09-13T00:41:25.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.063547+0000 mon.a (mon.0) 301 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:41:25.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.063586+0000 mon.a (mon.0) 302 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.063617+0000 mon.a (mon.0) 303 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.063650+0000 mon.a (mon.0) 304 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.063686+0000 mon.a (mon.0) 305 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.063729+0000 mon.a (mon.0) 306 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.063763+0000 mon.a (mon.0) 307 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.063796+0000 mon.a (mon.0) 308 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:25 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.063838+0000 mon.a (mon.0) 309 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:17.473526+0000 mon.c (mon.2) 2 : cluster [INF] mon.c calling monitor election 2024-09-13T00:41:25.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.000798+0000 mon.b (mon.1) 3 : cluster [INF] mon.b calling monitor election 2024-09-13T00:41:25.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.001042+0000 mon.a (mon.0) 293 : cluster [INF] mon.a calling monitor election 2024-09-13T00:41:25.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.024102+0000 mon.a (mon.0) 294 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-13T00:41:25.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.034277+0000 mon.a (mon.0) 295 : cluster [DBG] monmap e4: 3 mons at {a=[v2:172.21.15.77:3300/0,v1:172.21.15.77:6789/0],b=[v2:172.21.15.88:3300/0,v1:172.21.15.88:6789/0],c=[v2:172.21.15.77:3301/0,v1:172.21.15.77:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-13T00:41:25.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.055466+0000 mon.a (mon.0) 296 : cluster [DBG] fsmap foo:1 {0=foo.smithi088.krmowh=up:active} 1 up:standby 2024-09-13T00:41:25.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.055537+0000 mon.a (mon.0) 297 : cluster [DBG] osdmap e402: 8 total, 8 up, 8 in 2024-09-13T00:41:25.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.056204+0000 mon.a (mon.0) 298 : cluster [DBG] mgrmap e39: y(active, since 2m), standbys: x 2024-09-13T00:41:25.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.056507+0000 mon.a (mon.0) 299 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-09-13T00:41:25.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.063509+0000 mon.a (mon.0) 300 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-09-13T00:41:25.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.063547+0000 mon.a (mon.0) 301 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-09-13T00:41:25.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.063586+0000 mon.a (mon.0) 302 : cluster [ERR] osd.0's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.063617+0000 mon.a (mon.0) 303 : cluster [ERR] osd.1's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.063650+0000 mon.a (mon.0) 304 : cluster [ERR] osd.2's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.063686+0000 mon.a (mon.0) 305 : cluster [ERR] osd.3's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.063729+0000 mon.a (mon.0) 306 : cluster [ERR] osd.4's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.063763+0000 mon.a (mon.0) 307 : cluster [ERR] osd.5's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.063796+0000 mon.a (mon.0) 308 : cluster [ERR] osd.6's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:25.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:25 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.063838+0000 mon.a (mon.0) 309 : cluster [ERR] osd.7's public address is not in '172.21.0.1/32,172.21.0.0/20,172.21.15.254/32,172.21.0.2/32' subnet 2024-09-13T00:41:26.203 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.Clone_LargerDstObjSize (13778 ms) 2024-09-13T00:41:26.203 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.CloneFlatten_LargerDstObjSize 2024-09-13T00:41:26.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:26 smithi077 bash[83127]: cluster 2024-09-13T00:41:25.462124+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e403: 8 total, 8 up, 8 in 2024-09-13T00:41:26.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:26 smithi077 bash[83127]: cluster 2024-09-13T00:41:26.382173+0000 mgr.y (mgr.25304) 102 : cluster [DBG] pgmap v190: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:41:26.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:26 smithi077 bash[80681]: cluster 2024-09-13T00:41:25.462124+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e403: 8 total, 8 up, 8 in 2024-09-13T00:41:26.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:26 smithi077 bash[80681]: cluster 2024-09-13T00:41:26.382173+0000 mgr.y (mgr.25304) 102 : cluster [DBG] pgmap v190: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:41:26.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:26 smithi088 bash[68960]: cluster 2024-09-13T00:41:25.462124+0000 mon.a (mon.0) 310 : cluster [DBG] osdmap e403: 8 total, 8 up, 8 in 2024-09-13T00:41:26.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:26 smithi088 bash[68960]: cluster 2024-09-13T00:41:26.382173+0000 mgr.y (mgr.25304) 102 : cluster [DBG] pgmap v190: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:41:28.148 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:41:28.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:27 smithi077 bash[80681]: cluster 2024-09-13T00:41:26.980157+0000 mon.a (mon.0) 311 : cluster [DBG] osdmap e404: 8 total, 8 up, 8 in 2024-09-13T00:41:28.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:27 smithi077 bash[83127]: cluster 2024-09-13T00:41:26.980157+0000 mon.a (mon.0) 311 : cluster [DBG] osdmap e404: 8 total, 8 up, 8 in 2024-09-13T00:41:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:27 smithi088 bash[68960]: cluster 2024-09-13T00:41:26.980157+0000 mon.a (mon.0) 311 : cluster [DBG] osdmap e404: 8 total, 8 up, 8 in 2024-09-13T00:41:28.689 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.CloneFlatten_LargerDstObjSize (2485 ms) 2024-09-13T00:41:28.689 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.Stress_LargerDstObjSize 2024-09-13T00:41:28.739 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 28513927~1574574 2024-09-13T00:41:28.755 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 57169275~1517465 2024-09-13T00:41:28.757 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 54519070~5071057 2024-09-13T00:41:28.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:41:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:41:28] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.43.0" 2024-09-13T00:41:28.788 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 35951020~4841901 2024-09-13T00:41:28.789 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 65260760~4128839 2024-09-13T00:41:28.805 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 76153595~7732485 2024-09-13T00:41:28.806 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 75399509~1245325 2024-09-13T00:41:28.816 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 52724845~6967199 2024-09-13T00:41:28.824 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap0 2024-09-13T00:41:28.987 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:28 smithi088 bash[61791]: ts=2024-09-13T00:41:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:28 smithi077 bash[80681]: cluster 2024-09-13T00:41:27.985553+0000 mon.a (mon.0) 312 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2024-09-13T00:41:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:28 smithi077 bash[80681]: cluster 2024-09-13T00:41:28.382802+0000 mgr.y (mgr.25304) 103 : cluster [DBG] pgmap v193: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:41:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:28 smithi077 bash[80681]: audit 2024-09-13T00:41:28.411821+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:28 smithi077 bash[80681]: audit 2024-09-13T00:41:28.412122+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:28 smithi077 bash[80681]: audit 2024-09-13T00:41:28.420587+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:29.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:28 smithi077 bash[83127]: cluster 2024-09-13T00:41:27.985553+0000 mon.a (mon.0) 312 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2024-09-13T00:41:29.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:28 smithi077 bash[83127]: cluster 2024-09-13T00:41:28.382802+0000 mgr.y (mgr.25304) 103 : cluster [DBG] pgmap v193: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:41:29.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:28 smithi077 bash[83127]: audit 2024-09-13T00:41:28.411821+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:29.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:28 smithi077 bash[83127]: audit 2024-09-13T00:41:28.412122+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:29.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:28 smithi077 bash[83127]: audit 2024-09-13T00:41:28.420587+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:28 smithi088 bash[68960]: cluster 2024-09-13T00:41:27.985553+0000 mon.a (mon.0) 312 : cluster [DBG] osdmap e405: 8 total, 8 up, 8 in 2024-09-13T00:41:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:28 smithi088 bash[68960]: cluster 2024-09-13T00:41:28.382802+0000 mgr.y (mgr.25304) 103 : cluster [DBG] pgmap v193: 105 pgs: 105 active+clean; 394 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2024-09-13T00:41:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:28 smithi088 bash[68960]: audit 2024-09-13T00:41:28.411821+0000 mon.a (mon.0) 313 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:28 smithi088 bash[68960]: audit 2024-09-13T00:41:28.412122+0000 mon.a (mon.0) 314 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:29.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:28 smithi088 bash[68960]: audit 2024-09-13T00:41:28.420587+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:29.463 INFO:tasks.workunit.client.0.smithi077.stdout:clone image249 -> image250 2024-09-13T00:41:29.566 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 101278641 2024-09-13T00:41:29.581 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 46618053~7088055 2024-09-13T00:41:29.617 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 56262291~1525460 2024-09-13T00:41:29.626 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 11390781~1730668 2024-09-13T00:41:29.640 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 61467457~7635158 2024-09-13T00:41:29.653 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 50461389~3410388 2024-09-13T00:41:29.668 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 58605780~1803383 2024-09-13T00:41:29.671 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 30706434~2319660 2024-09-13T00:41:29.688 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 65805286~6941094 2024-09-13T00:41:29.694 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap1 2024-09-13T00:41:30.450 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 76035926~5610581 2024-09-13T00:41:30.456 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 4766945~6868747 2024-09-13T00:41:30.456 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 4632035~3574401 2024-09-13T00:41:30.470 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 92674401~5007802 2024-09-13T00:41:30.472 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 15789607~3825972 2024-09-13T00:41:30.472 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 99496227~339472 2024-09-13T00:41:30.485 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 16205258~8016344 2024-09-13T00:41:30.503 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 47566553~3734166 2024-09-13T00:41:30.525 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap2 2024-09-13T00:41:30.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:30 smithi077 bash[80681]: audit 2024-09-13T00:41:29.223394+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:30.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:30 smithi077 bash[80681]: audit 2024-09-13T00:41:29.235188+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:30.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:30 smithi077 bash[80681]: cluster 2024-09-13T00:41:29.427308+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2024-09-13T00:41:30.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:30 smithi077 bash[83127]: audit 2024-09-13T00:41:29.223394+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:30.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:30 smithi077 bash[83127]: audit 2024-09-13T00:41:29.235188+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:30.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:30 smithi077 bash[83127]: cluster 2024-09-13T00:41:29.427308+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2024-09-13T00:41:30.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:30 smithi088 bash[68960]: audit 2024-09-13T00:41:29.223394+0000 mon.a (mon.0) 316 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:30.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:30 smithi088 bash[68960]: audit 2024-09-13T00:41:29.235188+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:30.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:30 smithi088 bash[68960]: cluster 2024-09-13T00:41:29.427308+0000 mon.a (mon.0) 318 : cluster [DBG] osdmap e406: 8 total, 8 up, 8 in 2024-09-13T00:41:31.451 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 122211227 2024-09-13T00:41:31.473 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 109260981~4442335 2024-09-13T00:41:31.479 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 47289888~407840 2024-09-13T00:41:31.480 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 77870475~1853034 2024-09-13T00:41:31.480 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 28871306~7507081 2024-09-13T00:41:31.488 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 68308266~4383955 2024-09-13T00:41:31.488 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 86398655~449849 2024-09-13T00:41:31.499 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 116713493~5497734 2024-09-13T00:41:31.499 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 1190457~5604002 2024-09-13T00:41:31.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:31 smithi077 bash[83127]: cluster 2024-09-13T00:41:30.383464+0000 mgr.y (mgr.25304) 104 : cluster [DBG] pgmap v195: 105 pgs: 105 active+clean; 544 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 29 MiB/s wr, 363 op/s 2024-09-13T00:41:31.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:31 smithi077 bash[83127]: cluster 2024-09-13T00:41:30.429995+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2024-09-13T00:41:31.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:31 smithi077 bash[80681]: cluster 2024-09-13T00:41:30.383464+0000 mgr.y (mgr.25304) 104 : cluster [DBG] pgmap v195: 105 pgs: 105 active+clean; 544 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 29 MiB/s wr, 363 op/s 2024-09-13T00:41:31.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:31 smithi077 bash[80681]: cluster 2024-09-13T00:41:30.429995+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2024-09-13T00:41:31.540 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap3 2024-09-13T00:41:31.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:31 smithi088 bash[68960]: cluster 2024-09-13T00:41:30.383464+0000 mgr.y (mgr.25304) 104 : cluster [DBG] pgmap v195: 105 pgs: 105 active+clean; 544 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 38 MiB/s rd, 29 MiB/s wr, 363 op/s 2024-09-13T00:41:31.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:31 smithi088 bash[68960]: cluster 2024-09-13T00:41:30.429995+0000 mon.a (mon.0) 319 : cluster [DBG] osdmap e407: 8 total, 8 up, 8 in 2024-09-13T00:41:32.481 INFO:tasks.workunit.client.0.smithi077.stdout:clone image250 -> image251 2024-09-13T00:41:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:32 smithi088 bash[68960]: cluster 2024-09-13T00:41:31.435048+0000 mon.a (mon.0) 320 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2024-09-13T00:41:32.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:32 smithi088 bash[68960]: cluster 2024-09-13T00:41:32.384227+0000 mgr.y (mgr.25304) 105 : cluster [DBG] pgmap v198: 105 pgs: 105 active+clean; 544 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 33 MiB/s wr, 407 op/s 2024-09-13T00:41:32.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:32 smithi088 bash[61791]: ts=2024-09-13T00:41:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:32.630 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 172498743 2024-09-13T00:41:32.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:32 smithi077 bash[80681]: cluster 2024-09-13T00:41:31.435048+0000 mon.a (mon.0) 320 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2024-09-13T00:41:32.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:32 smithi077 bash[80681]: cluster 2024-09-13T00:41:32.384227+0000 mgr.y (mgr.25304) 105 : cluster [DBG] pgmap v198: 105 pgs: 105 active+clean; 544 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 33 MiB/s wr, 407 op/s 2024-09-13T00:41:32.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:32 smithi077 bash[83127]: cluster 2024-09-13T00:41:31.435048+0000 mon.a (mon.0) 320 : cluster [DBG] osdmap e408: 8 total, 8 up, 8 in 2024-09-13T00:41:32.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:32 smithi077 bash[83127]: cluster 2024-09-13T00:41:32.384227+0000 mgr.y (mgr.25304) 105 : cluster [DBG] pgmap v198: 105 pgs: 105 active+clean; 544 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 43 MiB/s rd, 33 MiB/s wr, 407 op/s 2024-09-13T00:41:33.184 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.Stress_LargerDstObjSize (4496 ms) 2024-09-13T00:41:33.184 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.NoSnaps_SmallerDstObjSize 2024-09-13T00:41:33.621 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.NoSnaps_SmallerDstObjSize (436 ms) 2024-09-13T00:41:33.621 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.Snaps_SmallerDstObjSize 2024-09-13T00:41:33.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:33 smithi077 bash[83127]: cluster 2024-09-13T00:41:32.444361+0000 mon.a (mon.0) 321 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2024-09-13T00:41:33.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:33 smithi077 bash[80681]: cluster 2024-09-13T00:41:32.444361+0000 mon.a (mon.0) 321 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2024-09-13T00:41:33.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:33 smithi088 bash[68960]: cluster 2024-09-13T00:41:32.444361+0000 mon.a (mon.0) 321 : cluster [DBG] osdmap e409: 8 total, 8 up, 8 in 2024-09-13T00:41:33.852 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:33 smithi088 bash[61791]: ts=2024-09-13T00:41:33.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:34.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:34 smithi088 bash[68960]: cluster 2024-09-13T00:41:34.385522+0000 mgr.y (mgr.25304) 106 : cluster [DBG] pgmap v200: 105 pgs: 105 active+clean; 665 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s rd, 19 MiB/s wr, 414 op/s 2024-09-13T00:41:34.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:34 smithi088 bash[61791]: ts=2024-09-13T00:41:34.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:34.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:34 smithi088 bash[61791]: ts=2024-09-13T00:41:34.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:34.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:34 smithi077 bash[80681]: cluster 2024-09-13T00:41:34.385522+0000 mgr.y (mgr.25304) 106 : cluster [DBG] pgmap v200: 105 pgs: 105 active+clean; 665 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s rd, 19 MiB/s wr, 414 op/s 2024-09-13T00:41:34.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:34 smithi077 bash[83127]: cluster 2024-09-13T00:41:34.385522+0000 mgr.y (mgr.25304) 106 : cluster [DBG] pgmap v200: 105 pgs: 105 active+clean; 665 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s rd, 19 MiB/s wr, 414 op/s 2024-09-13T00:41:35.586 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:35 smithi088 bash[68960]: cluster 2024-09-13T00:41:34.457313+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2024-09-13T00:41:35.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:35 smithi077 bash[80681]: cluster 2024-09-13T00:41:34.457313+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2024-09-13T00:41:35.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:35 smithi077 bash[83127]: cluster 2024-09-13T00:41:34.457313+0000 mon.a (mon.0) 322 : cluster [DBG] osdmap e410: 8 total, 8 up, 8 in 2024-09-13T00:41:36.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:36 smithi077 bash[83127]: cluster 2024-09-13T00:41:35.467370+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2024-09-13T00:41:36.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:36 smithi077 bash[83127]: cluster 2024-09-13T00:41:36.386171+0000 mgr.y (mgr.25304) 107 : cluster [DBG] pgmap v203: 105 pgs: 105 active+clean; 665 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s rd, 19 MiB/s wr, 415 op/s 2024-09-13T00:41:36.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:36 smithi077 bash[80681]: cluster 2024-09-13T00:41:35.467370+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2024-09-13T00:41:36.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:36 smithi077 bash[80681]: cluster 2024-09-13T00:41:36.386171+0000 mgr.y (mgr.25304) 107 : cluster [DBG] pgmap v203: 105 pgs: 105 active+clean; 665 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s rd, 19 MiB/s wr, 415 op/s 2024-09-13T00:41:36.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:36 smithi088 bash[68960]: cluster 2024-09-13T00:41:35.467370+0000 mon.a (mon.0) 323 : cluster [DBG] osdmap e411: 8 total, 8 up, 8 in 2024-09-13T00:41:36.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:36 smithi088 bash[68960]: cluster 2024-09-13T00:41:36.386171+0000 mgr.y (mgr.25304) 107 : cluster [DBG] pgmap v203: 105 pgs: 105 active+clean; 665 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s rd, 19 MiB/s wr, 415 op/s 2024-09-13T00:41:37.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:37 smithi077 bash[80681]: cluster 2024-09-13T00:41:36.477949+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2024-09-13T00:41:37.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:37 smithi077 bash[83127]: cluster 2024-09-13T00:41:36.477949+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2024-09-13T00:41:37.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:37 smithi088 bash[68960]: cluster 2024-09-13T00:41:36.477949+0000 mon.a (mon.0) 324 : cluster [DBG] osdmap e412: 8 total, 8 up, 8 in 2024-09-13T00:41:37.897 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.Snaps_SmallerDstObjSize (4277 ms) 2024-09-13T00:41:37.897 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.Clone_SmallerDstObjSize 2024-09-13T00:41:38.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:41:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:41:38] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.43.0" 2024-09-13T00:41:38.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:38 smithi077 bash[83127]: cluster 2024-09-13T00:41:37.485402+0000 mon.a (mon.0) 325 : cluster [DBG] osdmap e413: 8 total, 8 up, 8 in 2024-09-13T00:41:38.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:38 smithi077 bash[83127]: cluster 2024-09-13T00:41:38.386949+0000 mgr.y (mgr.25304) 108 : cluster [DBG] pgmap v206: 105 pgs: 105 active+clean; 665 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:41:38.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:38 smithi077 bash[80681]: cluster 2024-09-13T00:41:37.485402+0000 mon.a (mon.0) 325 : cluster [DBG] osdmap e413: 8 total, 8 up, 8 in 2024-09-13T00:41:38.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:38 smithi077 bash[80681]: cluster 2024-09-13T00:41:38.386949+0000 mgr.y (mgr.25304) 108 : cluster [DBG] pgmap v206: 105 pgs: 105 active+clean; 665 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:41:38.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:38 smithi088 bash[68960]: cluster 2024-09-13T00:41:37.485402+0000 mon.a (mon.0) 325 : cluster [DBG] osdmap e413: 8 total, 8 up, 8 in 2024-09-13T00:41:38.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:38 smithi088 bash[68960]: cluster 2024-09-13T00:41:38.386949+0000 mgr.y (mgr.25304) 108 : cluster [DBG] pgmap v206: 105 pgs: 105 active+clean; 665 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2024-09-13T00:41:38.852 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:38 smithi088 bash[61791]: ts=2024-09-13T00:41:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:39.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[80681]: cluster 2024-09-13T00:41:38.496359+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e414: 8 total, 8 up, 8 in 2024-09-13T00:41:39.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[80681]: audit 2024-09-13T00:41:38.641840+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:39.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[80681]: audit 2024-09-13T00:41:38.655457+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:39.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[80681]: audit 2024-09-13T00:41:38.657753+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:39.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[80681]: audit 2024-09-13T00:41:38.659755+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:41:39.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[80681]: audit 2024-09-13T00:41:38.671548+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:39.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[80681]: cephadm 2024-09-13T00:41:38.722235+0000 mgr.y (mgr.25304) 109 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-09-13T00:41:39.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[80681]: audit 2024-09-13T00:41:38.722802+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:41:39.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[80681]: audit 2024-09-13T00:41:38.724239+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:41:39.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[80681]: audit 2024-09-13T00:41:38.725463+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:39.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[80681]: cephadm 2024-09-13T00:41:38.727011+0000 mgr.y (mgr.25304) 110 : cephadm [INF] Reconfiguring daemon mon.a on smithi077 2024-09-13T00:41:39.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[83127]: cluster 2024-09-13T00:41:38.496359+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e414: 8 total, 8 up, 8 in 2024-09-13T00:41:39.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[83127]: audit 2024-09-13T00:41:38.641840+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:39.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[83127]: audit 2024-09-13T00:41:38.655457+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:39.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[83127]: audit 2024-09-13T00:41:38.657753+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:39.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[83127]: audit 2024-09-13T00:41:38.659755+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:41:39.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[83127]: audit 2024-09-13T00:41:38.671548+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:39.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[83127]: cephadm 2024-09-13T00:41:38.722235+0000 mgr.y (mgr.25304) 109 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-09-13T00:41:39.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[83127]: audit 2024-09-13T00:41:38.722802+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:41:39.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[83127]: audit 2024-09-13T00:41:38.724239+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:41:39.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[83127]: audit 2024-09-13T00:41:38.725463+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:39.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:39 smithi077 bash[83127]: cephadm 2024-09-13T00:41:38.727011+0000 mgr.y (mgr.25304) 110 : cephadm [INF] Reconfiguring daemon mon.a on smithi077 2024-09-13T00:41:39.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:39 smithi088 bash[68960]: cluster 2024-09-13T00:41:38.496359+0000 mon.a (mon.0) 326 : cluster [DBG] osdmap e414: 8 total, 8 up, 8 in 2024-09-13T00:41:39.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:39 smithi088 bash[68960]: audit 2024-09-13T00:41:38.641840+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:39.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:39 smithi088 bash[68960]: audit 2024-09-13T00:41:38.655457+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:39.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:39 smithi088 bash[68960]: audit 2024-09-13T00:41:38.657753+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:39.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:39 smithi088 bash[68960]: audit 2024-09-13T00:41:38.659755+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:41:39.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:39 smithi088 bash[68960]: audit 2024-09-13T00:41:38.671548+0000 mon.a (mon.0) 331 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:39.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:39 smithi088 bash[68960]: cephadm 2024-09-13T00:41:38.722235+0000 mgr.y (mgr.25304) 109 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-09-13T00:41:39.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:39 smithi088 bash[68960]: audit 2024-09-13T00:41:38.722802+0000 mon.a (mon.0) 332 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:41:39.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:39 smithi088 bash[68960]: audit 2024-09-13T00:41:38.724239+0000 mon.a (mon.0) 333 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:41:39.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:39 smithi088 bash[68960]: audit 2024-09-13T00:41:38.725463+0000 mon.a (mon.0) 334 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:39.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:39 smithi088 bash[68960]: cephadm 2024-09-13T00:41:38.727011+0000 mgr.y (mgr.25304) 110 : cephadm [INF] Reconfiguring daemon mon.a on smithi077 2024-09-13T00:41:40.346 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.Clone_SmallerDstObjSize (2449 ms) 2024-09-13T00:41:40.346 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.CloneFlatten_SmallerDstObjSize 2024-09-13T00:41:40.957 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:40 smithi077 bash[83127]: cluster 2024-09-13T00:41:39.675574+0000 mon.a (mon.0) 335 : cluster [DBG] osdmap e415: 8 total, 8 up, 8 in 2024-09-13T00:41:40.957 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:40 smithi077 bash[83127]: cluster 2024-09-13T00:41:40.387701+0000 mgr.y (mgr.25304) 111 : cluster [DBG] pgmap v209: 105 pgs: 105 active+clean; 709 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 222 op/s 2024-09-13T00:41:40.958 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:40 smithi077 bash[80681]: cluster 2024-09-13T00:41:39.675574+0000 mon.a (mon.0) 335 : cluster [DBG] osdmap e415: 8 total, 8 up, 8 in 2024-09-13T00:41:40.958 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:40 smithi077 bash[80681]: cluster 2024-09-13T00:41:40.387701+0000 mgr.y (mgr.25304) 111 : cluster [DBG] pgmap v209: 105 pgs: 105 active+clean; 709 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 222 op/s 2024-09-13T00:41:41.104 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:40 smithi088 bash[68960]: cluster 2024-09-13T00:41:39.675574+0000 mon.a (mon.0) 335 : cluster [DBG] osdmap e415: 8 total, 8 up, 8 in 2024-09-13T00:41:41.105 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:40 smithi088 bash[68960]: cluster 2024-09-13T00:41:40.387701+0000 mgr.y (mgr.25304) 111 : cluster [DBG] pgmap v209: 105 pgs: 105 active+clean; 709 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 222 op/s 2024-09-13T00:41:42.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:41 smithi077 bash[80681]: cluster 2024-09-13T00:41:40.685010+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e416: 8 total, 8 up, 8 in 2024-09-13T00:41:42.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:41 smithi077 bash[83127]: cluster 2024-09-13T00:41:40.685010+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e416: 8 total, 8 up, 8 in 2024-09-13T00:41:42.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:41 smithi088 bash[68960]: cluster 2024-09-13T00:41:40.685010+0000 mon.a (mon.0) 336 : cluster [DBG] osdmap e416: 8 total, 8 up, 8 in 2024-09-13T00:41:42.398 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.CloneFlatten_SmallerDstObjSize (2051 ms) 2024-09-13T00:41:42.398 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.Stress_SmallerDstObjSize 2024-09-13T00:41:42.460 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 73461232~3682077 2024-09-13T00:41:42.489 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 61441252~4519533 2024-09-13T00:41:42.489 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 56004378~1697926 2024-09-13T00:41:42.503 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 32328585~7916781 2024-09-13T00:41:42.504 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 45913548~2670183 2024-09-13T00:41:42.527 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 36632455~7212412 2024-09-13T00:41:42.529 INFO:tasks.workunit.client.0.smithi077.stdout:write: A 42619878~769321 2024-09-13T00:41:42.535 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 8276402~520885 2024-09-13T00:41:42.535 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap0 2024-09-13T00:41:42.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:42 smithi088 bash[61791]: ts=2024-09-13T00:41:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:42.890 INFO:tasks.workunit.client.0.smithi077.stdout:clone image270 -> image271 2024-09-13T00:41:43.022 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 16519816~8163555 2024-09-13T00:41:43.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[83127]: cluster 2024-09-13T00:41:41.687291+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e417: 8 total, 8 up, 8 in 2024-09-13T00:41:43.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[83127]: audit 2024-09-13T00:41:41.823554+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:43.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[83127]: audit 2024-09-13T00:41:41.833848+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:43.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[83127]: cephadm 2024-09-13T00:41:41.835182+0000 mgr.y (mgr.25304) 112 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-09-13T00:41:43.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[83127]: audit 2024-09-13T00:41:41.835604+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:41:43.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[83127]: audit 2024-09-13T00:41:41.836835+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:41:43.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[83127]: audit 2024-09-13T00:41:41.837954+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:43.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[83127]: cephadm 2024-09-13T00:41:41.839179+0000 mgr.y (mgr.25304) 113 : cephadm [INF] Reconfiguring daemon mgr.y on smithi077 2024-09-13T00:41:43.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[83127]: cluster 2024-09-13T00:41:42.388522+0000 mgr.y (mgr.25304) 114 : cluster [DBG] pgmap v212: 105 pgs: 105 active+clean; 709 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 222 op/s 2024-09-13T00:41:43.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[80681]: cluster 2024-09-13T00:41:41.687291+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e417: 8 total, 8 up, 8 in 2024-09-13T00:41:43.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[80681]: audit 2024-09-13T00:41:41.823554+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:43.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[80681]: audit 2024-09-13T00:41:41.833848+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:43.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[80681]: cephadm 2024-09-13T00:41:41.835182+0000 mgr.y (mgr.25304) 112 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-09-13T00:41:43.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[80681]: audit 2024-09-13T00:41:41.835604+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:41:43.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[80681]: audit 2024-09-13T00:41:41.836835+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:41:43.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[80681]: audit 2024-09-13T00:41:41.837954+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:43.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[80681]: cephadm 2024-09-13T00:41:41.839179+0000 mgr.y (mgr.25304) 113 : cephadm [INF] Reconfiguring daemon mgr.y on smithi077 2024-09-13T00:41:43.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:42 smithi077 bash[80681]: cluster 2024-09-13T00:41:42.388522+0000 mgr.y (mgr.25304) 114 : cluster [DBG] pgmap v212: 105 pgs: 105 active+clean; 709 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 222 op/s 2024-09-13T00:41:43.075 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 38827113~3640096 2024-09-13T00:41:43.091 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 67964354~4422973 2024-09-13T00:41:43.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:42 smithi088 bash[68960]: cluster 2024-09-13T00:41:41.687291+0000 mon.a (mon.0) 337 : cluster [DBG] osdmap e417: 8 total, 8 up, 8 in 2024-09-13T00:41:43.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:42 smithi088 bash[68960]: audit 2024-09-13T00:41:41.823554+0000 mon.a (mon.0) 338 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:43.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:42 smithi088 bash[68960]: audit 2024-09-13T00:41:41.833848+0000 mon.a (mon.0) 339 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:43.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:42 smithi088 bash[68960]: cephadm 2024-09-13T00:41:41.835182+0000 mgr.y (mgr.25304) 112 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-09-13T00:41:43.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:42 smithi088 bash[68960]: audit 2024-09-13T00:41:41.835604+0000 mon.a (mon.0) 340 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:41:43.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:42 smithi088 bash[68960]: audit 2024-09-13T00:41:41.836835+0000 mon.a (mon.0) 341 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:41:43.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:42 smithi088 bash[68960]: audit 2024-09-13T00:41:41.837954+0000 mon.a (mon.0) 342 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:43.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:42 smithi088 bash[68960]: cephadm 2024-09-13T00:41:41.839179+0000 mgr.y (mgr.25304) 113 : cephadm [INF] Reconfiguring daemon mgr.y on smithi077 2024-09-13T00:41:43.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:42 smithi088 bash[68960]: cluster 2024-09-13T00:41:42.388522+0000 mgr.y (mgr.25304) 114 : cluster [DBG] pgmap v212: 105 pgs: 105 active+clean; 709 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 222 op/s 2024-09-13T00:41:43.121 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 63009396~6144284 2024-09-13T00:41:43.129 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 20412298~3318265 2024-09-13T00:41:43.143 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 26384228~7631746 2024-09-13T00:41:43.153 INFO:tasks.workunit.client.0.smithi077.stdout:write: B 11313823~138271 2024-09-13T00:41:43.159 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 76315308~7570772 2024-09-13T00:41:43.175 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap1 2024-09-13T00:41:43.837 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:43 smithi088 bash[61791]: ts=2024-09-13T00:41:43.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:43.867 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 85828007 2024-09-13T00:41:43.880 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 30487395~5174385 2024-09-13T00:41:43.886 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 67865149~4612110 2024-09-13T00:41:43.891 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 6369106~6866656 2024-09-13T00:41:43.892 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 62300669~1525606 2024-09-13T00:41:43.901 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 77616989~8211018 2024-09-13T00:41:43.902 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 38823356~6836245 2024-09-13T00:41:43.920 INFO:tasks.workunit.client.0.smithi077.stdout:write: C 73687709~1188348 2024-09-13T00:41:43.932 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 65102058~8367347 2024-09-13T00:41:43.943 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap2 2024-09-13T00:41:44.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:43 smithi077 bash[80681]: cluster 2024-09-13T00:41:42.843641+0000 mon.a (mon.0) 343 : cluster [DBG] osdmap e418: 8 total, 8 up, 8 in 2024-09-13T00:41:44.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:43 smithi077 bash[80681]: audit 2024-09-13T00:41:43.411908+0000 mon.a (mon.0) 344 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:44.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:43 smithi077 bash[83127]: cluster 2024-09-13T00:41:42.843641+0000 mon.a (mon.0) 343 : cluster [DBG] osdmap e418: 8 total, 8 up, 8 in 2024-09-13T00:41:44.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:43 smithi077 bash[83127]: audit 2024-09-13T00:41:43.411908+0000 mon.a (mon.0) 344 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:44.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:43 smithi088 bash[68960]: cluster 2024-09-13T00:41:42.843641+0000 mon.a (mon.0) 343 : cluster [DBG] osdmap e418: 8 total, 8 up, 8 in 2024-09-13T00:41:44.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:43 smithi088 bash[68960]: audit 2024-09-13T00:41:43.411908+0000 mon.a (mon.0) 344 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:44.850 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:44 smithi088 bash[61791]: ts=2024-09-13T00:41:44.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:44.850 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:44 smithi088 bash[61791]: ts=2024-09-13T00:41:44.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:44.903 INFO:tasks.workunit.client.0.smithi077.stdout:clone image271 -> image272 2024-09-13T00:41:45.037 INFO:tasks.workunit.client.0.smithi077.stdout:resize: 166083858 2024-09-13T00:41:45.050 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 66249355~6902351 2024-09-13T00:41:45.080 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 90031569~7851788 2024-09-13T00:41:45.081 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 41597393~5370899 2024-09-13T00:41:45.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:44 smithi088 bash[68960]: cluster 2024-09-13T00:41:43.853039+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e419: 8 total, 8 up, 8 in 2024-09-13T00:41:45.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:44 smithi088 bash[68960]: cluster 2024-09-13T00:41:44.389799+0000 mgr.y (mgr.25304) 115 : cluster [DBG] pgmap v215: 105 pgs: 105 active+clean; 913 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 47 MiB/s rd, 43 MiB/s wr, 781 op/s 2024-09-13T00:41:45.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:44 smithi088 bash[68960]: audit 2024-09-13T00:41:44.710956+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:45.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:44 smithi088 bash[68960]: audit 2024-09-13T00:41:44.719761+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:45.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:44 smithi088 bash[68960]: audit 2024-09-13T00:41:44.721956+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:41:45.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:44 smithi088 bash[68960]: audit 2024-09-13T00:41:44.723743+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:41:45.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:44 smithi088 bash[68960]: audit 2024-09-13T00:41:44.725229+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:45.116 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:44 smithi077 bash[80681]: cluster 2024-09-13T00:41:43.853039+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e419: 8 total, 8 up, 8 in 2024-09-13T00:41:45.116 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:44 smithi077 bash[80681]: cluster 2024-09-13T00:41:44.389799+0000 mgr.y (mgr.25304) 115 : cluster [DBG] pgmap v215: 105 pgs: 105 active+clean; 913 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 47 MiB/s rd, 43 MiB/s wr, 781 op/s 2024-09-13T00:41:45.116 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:44 smithi077 bash[80681]: audit 2024-09-13T00:41:44.710956+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:45.117 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:44 smithi077 bash[80681]: audit 2024-09-13T00:41:44.719761+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:45.117 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:44 smithi077 bash[80681]: audit 2024-09-13T00:41:44.721956+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:41:45.117 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:44 smithi077 bash[80681]: audit 2024-09-13T00:41:44.723743+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:41:45.117 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:44 smithi077 bash[80681]: audit 2024-09-13T00:41:44.725229+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:45.117 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:44 smithi077 bash[83127]: cluster 2024-09-13T00:41:43.853039+0000 mon.a (mon.0) 345 : cluster [DBG] osdmap e419: 8 total, 8 up, 8 in 2024-09-13T00:41:45.117 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:44 smithi077 bash[83127]: cluster 2024-09-13T00:41:44.389799+0000 mgr.y (mgr.25304) 115 : cluster [DBG] pgmap v215: 105 pgs: 105 active+clean; 913 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 47 MiB/s rd, 43 MiB/s wr, 781 op/s 2024-09-13T00:41:45.117 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:44 smithi077 bash[83127]: audit 2024-09-13T00:41:44.710956+0000 mon.a (mon.0) 346 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:45.117 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:44 smithi077 bash[83127]: audit 2024-09-13T00:41:44.719761+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:45.117 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:44 smithi077 bash[83127]: audit 2024-09-13T00:41:44.721956+0000 mon.a (mon.0) 348 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:41:45.118 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:44 smithi077 bash[83127]: audit 2024-09-13T00:41:44.723743+0000 mon.a (mon.0) 349 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:41:45.118 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:44 smithi077 bash[83127]: audit 2024-09-13T00:41:44.725229+0000 mon.a (mon.0) 350 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:45.118 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 73456034~682097 2024-09-13T00:41:45.118 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 123768821~4259255 2024-09-13T00:41:45.137 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 160625632~5433155 2024-09-13T00:41:45.138 INFO:tasks.workunit.client.0.smithi077.stdout:write: D 141099233~3114086 2024-09-13T00:41:45.152 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 17323885~2424684 2024-09-13T00:41:45.173 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap3 2024-09-13T00:41:46.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:45 smithi077 bash[80681]: cephadm 2024-09-13T00:41:44.721371+0000 mgr.y (mgr.25304) 116 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-13T00:41:46.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:45 smithi077 bash[80681]: cephadm 2024-09-13T00:41:44.726756+0000 mgr.y (mgr.25304) 117 : cephadm [INF] Reconfiguring daemon mon.c on smithi077 2024-09-13T00:41:46.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:45 smithi077 bash[80681]: cluster 2024-09-13T00:41:44.870725+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e420: 8 total, 8 up, 8 in 2024-09-13T00:41:46.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:45 smithi077 bash[83127]: cephadm 2024-09-13T00:41:44.721371+0000 mgr.y (mgr.25304) 116 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-13T00:41:46.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:45 smithi077 bash[83127]: cephadm 2024-09-13T00:41:44.726756+0000 mgr.y (mgr.25304) 117 : cephadm [INF] Reconfiguring daemon mon.c on smithi077 2024-09-13T00:41:46.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:45 smithi077 bash[83127]: cluster 2024-09-13T00:41:44.870725+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e420: 8 total, 8 up, 8 in 2024-09-13T00:41:46.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:45 smithi088 bash[68960]: cephadm 2024-09-13T00:41:44.721371+0000 mgr.y (mgr.25304) 116 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-09-13T00:41:46.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:45 smithi088 bash[68960]: cephadm 2024-09-13T00:41:44.726756+0000 mgr.y (mgr.25304) 117 : cephadm [INF] Reconfiguring daemon mon.c on smithi077 2024-09-13T00:41:46.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:45 smithi088 bash[68960]: cluster 2024-09-13T00:41:44.870725+0000 mon.a (mon.0) 351 : cluster [DBG] osdmap e420: 8 total, 8 up, 8 in 2024-09-13T00:41:47.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:46 smithi077 bash[80681]: cluster 2024-09-13T00:41:45.878401+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e421: 8 total, 8 up, 8 in 2024-09-13T00:41:47.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:46 smithi077 bash[80681]: cluster 2024-09-13T00:41:46.390230+0000 mgr.y (mgr.25304) 118 : cluster [DBG] pgmap v218: 105 pgs: 105 active+clean; 913 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 47 MiB/s rd, 43 MiB/s wr, 781 op/s 2024-09-13T00:41:47.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:46 smithi077 bash[83127]: cluster 2024-09-13T00:41:45.878401+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e421: 8 total, 8 up, 8 in 2024-09-13T00:41:47.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:46 smithi077 bash[83127]: cluster 2024-09-13T00:41:46.390230+0000 mgr.y (mgr.25304) 118 : cluster [DBG] pgmap v218: 105 pgs: 105 active+clean; 913 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 47 MiB/s rd, 43 MiB/s wr, 781 op/s 2024-09-13T00:41:47.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:46 smithi088 bash[68960]: cluster 2024-09-13T00:41:45.878401+0000 mon.a (mon.0) 352 : cluster [DBG] osdmap e421: 8 total, 8 up, 8 in 2024-09-13T00:41:47.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:46 smithi088 bash[68960]: cluster 2024-09-13T00:41:46.390230+0000 mgr.y (mgr.25304) 118 : cluster [DBG] pgmap v218: 105 pgs: 105 active+clean; 913 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 47 MiB/s rd, 43 MiB/s wr, 781 op/s 2024-09-13T00:41:47.584 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.Stress_SmallerDstObjSize (5187 ms) 2024-09-13T00:41:47.584 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize 2024-09-13T00:41:47.642 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:41:47.782 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize (197 ms) 2024-09-13T00:41:47.782 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingLargerDstObjSize 2024-09-13T00:41:47.832 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:41:47.995 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.Snaps_StrippingLargerDstObjSize (214 ms) 2024-09-13T00:41:47.995 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.Clone_StrippingLargerDstObjSize 2024-09-13T00:41:48.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:47 smithi077 bash[83127]: cluster 2024-09-13T00:41:46.888931+0000 mon.a (mon.0) 353 : cluster [DBG] osdmap e422: 8 total, 8 up, 8 in 2024-09-13T00:41:48.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:47 smithi077 bash[83127]: audit 2024-09-13T00:41:47.319624+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:48.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:47 smithi077 bash[83127]: audit 2024-09-13T00:41:47.328683+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:48.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:47 smithi077 bash[83127]: cephadm 2024-09-13T00:41:47.329805+0000 mgr.y (mgr.25304) 119 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-09-13T00:41:48.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:47 smithi077 bash[83127]: audit 2024-09-13T00:41:47.330252+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T00:41:48.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:47 smithi077 bash[83127]: audit 2024-09-13T00:41:47.331595+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:48.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:47 smithi077 bash[83127]: cephadm 2024-09-13T00:41:47.334379+0000 mgr.y (mgr.25304) 120 : cephadm [INF] Reconfiguring daemon osd.0 on smithi077 2024-09-13T00:41:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:47 smithi077 bash[80681]: cluster 2024-09-13T00:41:46.888931+0000 mon.a (mon.0) 353 : cluster [DBG] osdmap e422: 8 total, 8 up, 8 in 2024-09-13T00:41:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:47 smithi077 bash[80681]: audit 2024-09-13T00:41:47.319624+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:47 smithi077 bash[80681]: audit 2024-09-13T00:41:47.328683+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:47 smithi077 bash[80681]: cephadm 2024-09-13T00:41:47.329805+0000 mgr.y (mgr.25304) 119 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-09-13T00:41:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:47 smithi077 bash[80681]: audit 2024-09-13T00:41:47.330252+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T00:41:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:47 smithi077 bash[80681]: audit 2024-09-13T00:41:47.331595+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:48.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:47 smithi077 bash[80681]: cephadm 2024-09-13T00:41:47.334379+0000 mgr.y (mgr.25304) 120 : cephadm [INF] Reconfiguring daemon osd.0 on smithi077 2024-09-13T00:41:48.045 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:41:48.211 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.Clone_StrippingLargerDstObjSize (216 ms) 2024-09-13T00:41:48.211 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize 2024-09-13T00:41:48.271 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:41:48.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:47 smithi088 bash[68960]: cluster 2024-09-13T00:41:46.888931+0000 mon.a (mon.0) 353 : cluster [DBG] osdmap e422: 8 total, 8 up, 8 in 2024-09-13T00:41:48.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:47 smithi088 bash[68960]: audit 2024-09-13T00:41:47.319624+0000 mon.a (mon.0) 354 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:48.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:47 smithi088 bash[68960]: audit 2024-09-13T00:41:47.328683+0000 mon.a (mon.0) 355 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:48.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:47 smithi088 bash[68960]: cephadm 2024-09-13T00:41:47.329805+0000 mgr.y (mgr.25304) 119 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-09-13T00:41:48.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:47 smithi088 bash[68960]: audit 2024-09-13T00:41:47.330252+0000 mon.a (mon.0) 356 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T00:41:48.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:47 smithi088 bash[68960]: audit 2024-09-13T00:41:47.331595+0000 mon.a (mon.0) 357 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:48.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:47 smithi088 bash[68960]: cephadm 2024-09-13T00:41:47.334379+0000 mgr.y (mgr.25304) 120 : cephadm [INF] Reconfiguring daemon osd.0 on smithi077 2024-09-13T00:41:48.444 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize (232 ms) 2024-09-13T00:41:48.444 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.Stress_StrippingLargerDstObjSize 2024-09-13T00:41:48.502 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:41:48.671 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.Stress_StrippingLargerDstObjSize (228 ms) 2024-09-13T00:41:48.672 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize 2024-09-13T00:41:48.723 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:41:48.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:41:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:41:48] "GET /metrics HTTP/1.1" 200 36452 "" "Prometheus/2.43.0" 2024-09-13T00:41:48.884 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize (213 ms) 2024-09-13T00:41:48.885 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize 2024-09-13T00:41:48.946 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:41:49.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:48 smithi088 bash[68960]: cluster 2024-09-13T00:41:48.390976+0000 mgr.y (mgr.25304) 121 : cluster [DBG] pgmap v220: 105 pgs: 105 active+clean; 913 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 41 MiB/s rd, 38 MiB/s wr, 688 op/s 2024-09-13T00:41:49.101 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:48 smithi088 bash[61791]: ts=2024-09-13T00:41:48.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:49.111 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize (227 ms) 2024-09-13T00:41:49.111 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.Clone_StrippingSmallerDstObjSize 2024-09-13T00:41:49.160 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:41:49.164 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:48 smithi077 bash[83127]: cluster 2024-09-13T00:41:48.390976+0000 mgr.y (mgr.25304) 121 : cluster [DBG] pgmap v220: 105 pgs: 105 active+clean; 913 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 41 MiB/s rd, 38 MiB/s wr, 688 op/s 2024-09-13T00:41:49.165 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:48 smithi077 bash[80681]: cluster 2024-09-13T00:41:48.390976+0000 mgr.y (mgr.25304) 121 : cluster [DBG] pgmap v220: 105 pgs: 105 active+clean; 913 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 41 MiB/s rd, 38 MiB/s wr, 688 op/s 2024-09-13T00:41:49.266 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.Clone_StrippingSmallerDstObjSize (155 ms) 2024-09-13T00:41:49.266 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize 2024-09-13T00:41:49.296 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:41:49.393 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize (127 ms) 2024-09-13T00:41:49.393 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestDeepCopy.Stress_StrippingSmallerDstObjSize 2024-09-13T00:41:49.439 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:41:49.579 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestDeepCopy.Stress_StrippingSmallerDstObjSize (185 ms) 2024-09-13T00:41:49.909 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 31 tests from TestDeepCopy (78637 ms total) 2024-09-13T00:41:49.909 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:41:49.909 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 51 tests from TestMirroring 2024-09-13T00:41:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:50 smithi077 bash[80681]: cluster 2024-09-13T00:41:49.903457+0000 mon.a (mon.0) 358 : cluster [DBG] osdmap e423: 8 total, 8 up, 8 in 2024-09-13T00:41:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:50 smithi077 bash[80681]: audit 2024-09-13T00:41:50.385811+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:50 smithi077 bash[80681]: cluster 2024-09-13T00:41:50.391600+0000 mgr.y (mgr.25304) 122 : cluster [DBG] pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:41:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:50 smithi077 bash[80681]: audit 2024-09-13T00:41:50.394226+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:50 smithi077 bash[80681]: cephadm 2024-09-13T00:41:50.396019+0000 mgr.y (mgr.25304) 123 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-09-13T00:41:51.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:50 smithi077 bash[80681]: audit 2024-09-13T00:41:50.396430+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T00:41:51.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:50 smithi077 bash[80681]: audit 2024-09-13T00:41:50.397459+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:51.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:50 smithi077 bash[80681]: cephadm 2024-09-13T00:41:50.399973+0000 mgr.y (mgr.25304) 124 : cephadm [INF] Reconfiguring daemon osd.1 on smithi077 2024-09-13T00:41:51.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:50 smithi077 bash[83127]: cluster 2024-09-13T00:41:49.903457+0000 mon.a (mon.0) 358 : cluster [DBG] osdmap e423: 8 total, 8 up, 8 in 2024-09-13T00:41:51.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:50 smithi077 bash[83127]: audit 2024-09-13T00:41:50.385811+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:51.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:50 smithi077 bash[83127]: cluster 2024-09-13T00:41:50.391600+0000 mgr.y (mgr.25304) 122 : cluster [DBG] pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:41:51.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:50 smithi077 bash[83127]: audit 2024-09-13T00:41:50.394226+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:51.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:50 smithi077 bash[83127]: cephadm 2024-09-13T00:41:50.396019+0000 mgr.y (mgr.25304) 123 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-09-13T00:41:51.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:50 smithi077 bash[83127]: audit 2024-09-13T00:41:50.396430+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T00:41:51.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:50 smithi077 bash[83127]: audit 2024-09-13T00:41:50.397459+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:51.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:50 smithi077 bash[83127]: cephadm 2024-09-13T00:41:50.399973+0000 mgr.y (mgr.25304) 124 : cephadm [INF] Reconfiguring daemon osd.1 on smithi077 2024-09-13T00:41:51.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:50 smithi088 bash[68960]: cluster 2024-09-13T00:41:49.903457+0000 mon.a (mon.0) 358 : cluster [DBG] osdmap e423: 8 total, 8 up, 8 in 2024-09-13T00:41:51.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:50 smithi088 bash[68960]: audit 2024-09-13T00:41:50.385811+0000 mon.a (mon.0) 359 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:51.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:50 smithi088 bash[68960]: cluster 2024-09-13T00:41:50.391600+0000 mgr.y (mgr.25304) 122 : cluster [DBG] pgmap v222: 73 pgs: 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:41:51.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:50 smithi088 bash[68960]: audit 2024-09-13T00:41:50.394226+0000 mon.a (mon.0) 360 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:51.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:50 smithi088 bash[68960]: cephadm 2024-09-13T00:41:50.396019+0000 mgr.y (mgr.25304) 123 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-09-13T00:41:51.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:50 smithi088 bash[68960]: audit 2024-09-13T00:41:50.396430+0000 mon.a (mon.0) 361 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T00:41:51.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:50 smithi088 bash[68960]: audit 2024-09-13T00:41:50.397459+0000 mon.a (mon.0) 362 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:51.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:50 smithi088 bash[68960]: cephadm 2024-09-13T00:41:50.399973+0000 mgr.y (mgr.25304) 124 : cephadm [INF] Reconfiguring daemon osd.1 on smithi077 2024-09-13T00:41:51.921 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage 2024-09-13T00:41:52.266 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:51 smithi077 bash[80681]: cluster 2024-09-13T00:41:50.917900+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e424: 8 total, 8 up, 8 in 2024-09-13T00:41:52.266 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:51 smithi077 bash[80681]: audit 2024-09-13T00:41:50.919978+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:41:52.266 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:51 smithi077 bash[80681]: audit 2024-09-13T00:41:50.920671+0000 mon.a (mon.0) 364 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:41:52.266 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:51 smithi077 bash[83127]: cluster 2024-09-13T00:41:50.917900+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e424: 8 total, 8 up, 8 in 2024-09-13T00:41:52.266 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:51 smithi077 bash[83127]: audit 2024-09-13T00:41:50.919978+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:41:52.266 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:51 smithi077 bash[83127]: audit 2024-09-13T00:41:50.920671+0000 mon.a (mon.0) 364 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:41:52.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:51 smithi088 bash[68960]: cluster 2024-09-13T00:41:50.917900+0000 mon.a (mon.0) 363 : cluster [DBG] osdmap e424: 8 total, 8 up, 8 in 2024-09-13T00:41:52.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:51 smithi088 bash[68960]: audit 2024-09-13T00:41:50.919978+0000 mon.c (mon.2) 3 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:41:52.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:51 smithi088 bash[68960]: audit 2024-09-13T00:41:50.920671+0000 mon.a (mon.0) 364 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:41:52.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:52 smithi088 bash[61791]: ts=2024-09-13T00:41:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:53.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:52 smithi077 bash[83127]: audit 2024-09-13T00:41:51.920298+0000 mon.a (mon.0) 365 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:41:53.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:52 smithi077 bash[83127]: cluster 2024-09-13T00:41:51.921516+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e425: 8 total, 8 up, 8 in 2024-09-13T00:41:53.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:52 smithi077 bash[83127]: cluster 2024-09-13T00:41:52.392281+0000 mgr.y (mgr.25304) 125 : cluster [DBG] pgmap v225: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:41:53.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:52 smithi077 bash[80681]: audit 2024-09-13T00:41:51.920298+0000 mon.a (mon.0) 365 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:41:53.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:52 smithi077 bash[80681]: cluster 2024-09-13T00:41:51.921516+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e425: 8 total, 8 up, 8 in 2024-09-13T00:41:53.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:52 smithi077 bash[80681]: cluster 2024-09-13T00:41:52.392281+0000 mgr.y (mgr.25304) 125 : cluster [DBG] pgmap v225: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:41:53.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:52 smithi088 bash[68960]: audit 2024-09-13T00:41:51.920298+0000 mon.a (mon.0) 365 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:41:53.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:52 smithi088 bash[68960]: cluster 2024-09-13T00:41:51.921516+0000 mon.a (mon.0) 366 : cluster [DBG] osdmap e425: 8 total, 8 up, 8 in 2024-09-13T00:41:53.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:52 smithi088 bash[68960]: cluster 2024-09-13T00:41:52.392281+0000 mgr.y (mgr.25304) 125 : cluster [DBG] pgmap v225: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2024-09-13T00:41:53.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:53 smithi088 bash[61791]: ts=2024-09-13T00:41:53.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:54.143 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage (2223 ms) 2024-09-13T00:41:54.144 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool 2024-09-13T00:41:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:53 smithi077 bash[80681]: cluster 2024-09-13T00:41:52.942650+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e426: 8 total, 8 up, 8 in 2024-09-13T00:41:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:53 smithi077 bash[80681]: audit 2024-09-13T00:41:53.175623+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:53 smithi077 bash[80681]: audit 2024-09-13T00:41:53.182100+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:53 smithi077 bash[80681]: cephadm 2024-09-13T00:41:53.183182+0000 mgr.y (mgr.25304) 126 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-09-13T00:41:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:53 smithi077 bash[80681]: audit 2024-09-13T00:41:53.183559+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T00:41:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:53 smithi077 bash[80681]: audit 2024-09-13T00:41:53.184754+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:53 smithi077 bash[80681]: cephadm 2024-09-13T00:41:53.187561+0000 mgr.y (mgr.25304) 127 : cephadm [INF] Reconfiguring daemon osd.2 on smithi077 2024-09-13T00:41:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:53 smithi077 bash[83127]: cluster 2024-09-13T00:41:52.942650+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e426: 8 total, 8 up, 8 in 2024-09-13T00:41:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:53 smithi077 bash[83127]: audit 2024-09-13T00:41:53.175623+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:53 smithi077 bash[83127]: audit 2024-09-13T00:41:53.182100+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:53 smithi077 bash[83127]: cephadm 2024-09-13T00:41:53.183182+0000 mgr.y (mgr.25304) 126 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-09-13T00:41:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:53 smithi077 bash[83127]: audit 2024-09-13T00:41:53.183559+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T00:41:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:53 smithi077 bash[83127]: audit 2024-09-13T00:41:53.184754+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:53 smithi077 bash[83127]: cephadm 2024-09-13T00:41:53.187561+0000 mgr.y (mgr.25304) 127 : cephadm [INF] Reconfiguring daemon osd.2 on smithi077 2024-09-13T00:41:54.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:53 smithi088 bash[68960]: cluster 2024-09-13T00:41:52.942650+0000 mon.a (mon.0) 367 : cluster [DBG] osdmap e426: 8 total, 8 up, 8 in 2024-09-13T00:41:54.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:53 smithi088 bash[68960]: audit 2024-09-13T00:41:53.175623+0000 mon.a (mon.0) 368 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:54.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:53 smithi088 bash[68960]: audit 2024-09-13T00:41:53.182100+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:54.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:53 smithi088 bash[68960]: cephadm 2024-09-13T00:41:53.183182+0000 mgr.y (mgr.25304) 126 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-09-13T00:41:54.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:53 smithi088 bash[68960]: audit 2024-09-13T00:41:53.183559+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T00:41:54.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:53 smithi088 bash[68960]: audit 2024-09-13T00:41:53.184754+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:54.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:53 smithi088 bash[68960]: cephadm 2024-09-13T00:41:53.187561+0000 mgr.y (mgr.25304) 127 : cephadm [INF] Reconfiguring daemon osd.2 on smithi077 2024-09-13T00:41:54.386 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool (242 ms) 2024-09-13T00:41:54.386 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled 2024-09-13T00:41:54.587 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled (201 ms) 2024-09-13T00:41:54.587 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage 2024-09-13T00:41:54.813 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage (226 ms) 2024-09-13T00:41:54.813 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap 2024-09-13T00:41:54.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:54 smithi088 bash[61791]: ts=2024-09-13T00:41:54.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:54.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:54 smithi088 bash[61791]: ts=2024-09-13T00:41:54.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:54.987 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap (175 ms) 2024-09-13T00:41:54.987 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModePool 2024-09-13T00:41:55.160 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModePool (173 ms) 2024-09-13T00:41:55.160 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled 2024-09-13T00:41:55.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:54 smithi077 bash[83127]: cluster 2024-09-13T00:41:53.953278+0000 mon.a (mon.0) 372 : cluster [DBG] osdmap e427: 8 total, 8 up, 8 in 2024-09-13T00:41:55.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:54 smithi077 bash[83127]: cluster 2024-09-13T00:41:54.393542+0000 mgr.y (mgr.25304) 128 : cluster [DBG] pgmap v228: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 13 KiB/s wr, 118 op/s 2024-09-13T00:41:55.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:54 smithi077 bash[80681]: cluster 2024-09-13T00:41:53.953278+0000 mon.a (mon.0) 372 : cluster [DBG] osdmap e427: 8 total, 8 up, 8 in 2024-09-13T00:41:55.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:54 smithi077 bash[80681]: cluster 2024-09-13T00:41:54.393542+0000 mgr.y (mgr.25304) 128 : cluster [DBG] pgmap v228: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 13 KiB/s wr, 118 op/s 2024-09-13T00:41:55.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:54 smithi088 bash[68960]: cluster 2024-09-13T00:41:53.953278+0000 mon.a (mon.0) 372 : cluster [DBG] osdmap e427: 8 total, 8 up, 8 in 2024-09-13T00:41:55.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:54 smithi088 bash[68960]: cluster 2024-09-13T00:41:54.393542+0000 mgr.y (mgr.25304) 128 : cluster [DBG] pgmap v228: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 13 KiB/s wr, 118 op/s 2024-09-13T00:41:55.403 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled (242 ms) 2024-09-13T00:41:55.403 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.DisableImageMirrorWithPeer 2024-09-13T00:41:55.403 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:41:55.403 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.DisableImageMirrorWithPeer (0 ms) 2024-09-13T00:41:55.404 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.DisableJournalingWithPeer 2024-09-13T00:41:55.404 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:41:55.404 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.DisableJournalingWithPeer (0 ms) 2024-09-13T00:41:55.404 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling 2024-09-13T00:41:55.587 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling (185 ms) 2024-09-13T00:41:55.587 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling 2024-09-13T00:41:55.804 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling (216 ms) 2024-09-13T00:41:55.804 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling 2024-09-13T00:41:57.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:56 smithi077 bash[83127]: cluster 2024-09-13T00:41:55.875284+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e428: 8 total, 8 up, 8 in 2024-09-13T00:41:57.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:56 smithi077 bash[83127]: audit 2024-09-13T00:41:56.100507+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:57.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:56 smithi077 bash[83127]: audit 2024-09-13T00:41:56.109174+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:57.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:56 smithi077 bash[83127]: cephadm 2024-09-13T00:41:56.110513+0000 mgr.y (mgr.25304) 129 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-09-13T00:41:57.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:56 smithi077 bash[83127]: audit 2024-09-13T00:41:56.110885+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T00:41:57.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:56 smithi077 bash[83127]: audit 2024-09-13T00:41:56.111987+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:57.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:56 smithi077 bash[83127]: cephadm 2024-09-13T00:41:56.114684+0000 mgr.y (mgr.25304) 130 : cephadm [INF] Reconfiguring daemon osd.3 on smithi077 2024-09-13T00:41:57.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:56 smithi077 bash[83127]: cluster 2024-09-13T00:41:56.393985+0000 mgr.y (mgr.25304) 131 : cluster [DBG] pgmap v230: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 11 KiB/s wr, 106 op/s 2024-09-13T00:41:57.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:56 smithi077 bash[80681]: cluster 2024-09-13T00:41:55.875284+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e428: 8 total, 8 up, 8 in 2024-09-13T00:41:57.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:56 smithi077 bash[80681]: audit 2024-09-13T00:41:56.100507+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:57.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:56 smithi077 bash[80681]: audit 2024-09-13T00:41:56.109174+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:57.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:56 smithi077 bash[80681]: cephadm 2024-09-13T00:41:56.110513+0000 mgr.y (mgr.25304) 129 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-09-13T00:41:57.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:56 smithi077 bash[80681]: audit 2024-09-13T00:41:56.110885+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T00:41:57.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:56 smithi077 bash[80681]: audit 2024-09-13T00:41:56.111987+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:57.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:56 smithi077 bash[80681]: cephadm 2024-09-13T00:41:56.114684+0000 mgr.y (mgr.25304) 130 : cephadm [INF] Reconfiguring daemon osd.3 on smithi077 2024-09-13T00:41:57.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:56 smithi077 bash[80681]: cluster 2024-09-13T00:41:56.393985+0000 mgr.y (mgr.25304) 131 : cluster [DBG] pgmap v230: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 11 KiB/s wr, 106 op/s 2024-09-13T00:41:57.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:56 smithi088 bash[68960]: cluster 2024-09-13T00:41:55.875284+0000 mon.a (mon.0) 373 : cluster [DBG] osdmap e428: 8 total, 8 up, 8 in 2024-09-13T00:41:57.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:56 smithi088 bash[68960]: audit 2024-09-13T00:41:56.100507+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:57.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:56 smithi088 bash[68960]: audit 2024-09-13T00:41:56.109174+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:57.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:56 smithi088 bash[68960]: cephadm 2024-09-13T00:41:56.110513+0000 mgr.y (mgr.25304) 129 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-09-13T00:41:57.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:56 smithi088 bash[68960]: audit 2024-09-13T00:41:56.110885+0000 mon.a (mon.0) 376 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T00:41:57.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:56 smithi088 bash[68960]: audit 2024-09-13T00:41:56.111987+0000 mon.a (mon.0) 377 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:57.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:56 smithi088 bash[68960]: cephadm 2024-09-13T00:41:56.114684+0000 mgr.y (mgr.25304) 130 : cephadm [INF] Reconfiguring daemon osd.3 on smithi077 2024-09-13T00:41:57.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:56 smithi088 bash[68960]: cluster 2024-09-13T00:41:56.393985+0000 mgr.y (mgr.25304) 131 : cluster [DBG] pgmap v230: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 11 KiB/s wr, 106 op/s 2024-09-13T00:41:57.980 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling (2176 ms) 2024-09-13T00:41:57.980 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock 2024-09-13T00:41:58.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:57 smithi077 bash[80681]: cluster 2024-09-13T00:41:56.879118+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e429: 8 total, 8 up, 8 in 2024-09-13T00:41:58.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:57 smithi077 bash[83127]: cluster 2024-09-13T00:41:56.879118+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e429: 8 total, 8 up, 8 in 2024-09-13T00:41:58.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:57 smithi088 bash[68960]: cluster 2024-09-13T00:41:56.879118+0000 mon.a (mon.0) 378 : cluster [DBG] osdmap e429: 8 total, 8 up, 8 in 2024-09-13T00:41:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:41:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:41:58] "GET /metrics HTTP/1.1" 200 36419 "" "Prometheus/2.43.0" 2024-09-13T00:41:59.101 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:41:58 smithi088 bash[61791]: ts=2024-09-13T00:41:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:41:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:58 smithi088 bash[68960]: cluster 2024-09-13T00:41:57.895246+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e430: 8 total, 8 up, 8 in 2024-09-13T00:41:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:58 smithi088 bash[68960]: cluster 2024-09-13T00:41:58.394656+0000 mgr.y (mgr.25304) 132 : cluster [DBG] pgmap v233: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s rd, 11 KiB/s wr, 106 op/s 2024-09-13T00:41:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:58 smithi088 bash[68960]: audit 2024-09-13T00:41:58.412107+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:58 smithi088 bash[68960]: audit 2024-09-13T00:41:58.850159+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:58 smithi088 bash[68960]: audit 2024-09-13T00:41:58.855675+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:58 smithi088 bash[68960]: audit 2024-09-13T00:41:58.857576+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi077.bigbbf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:41:59.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:58 smithi088 bash[68960]: audit 2024-09-13T00:41:58.858513+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:59.242 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:58 smithi077 bash[80681]: cluster 2024-09-13T00:41:57.895246+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e430: 8 total, 8 up, 8 in 2024-09-13T00:41:59.243 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:58 smithi077 bash[80681]: cluster 2024-09-13T00:41:58.394656+0000 mgr.y (mgr.25304) 132 : cluster [DBG] pgmap v233: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s rd, 11 KiB/s wr, 106 op/s 2024-09-13T00:41:59.243 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:58 smithi077 bash[80681]: audit 2024-09-13T00:41:58.412107+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:59.243 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:58 smithi077 bash[80681]: audit 2024-09-13T00:41:58.850159+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:59.243 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:58 smithi077 bash[80681]: audit 2024-09-13T00:41:58.855675+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:59.243 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:58 smithi077 bash[80681]: audit 2024-09-13T00:41:58.857576+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi077.bigbbf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:41:59.243 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:58 smithi077 bash[80681]: audit 2024-09-13T00:41:58.858513+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:41:59.243 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:58 smithi077 bash[83127]: cluster 2024-09-13T00:41:57.895246+0000 mon.a (mon.0) 379 : cluster [DBG] osdmap e430: 8 total, 8 up, 8 in 2024-09-13T00:41:59.243 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:58 smithi077 bash[83127]: cluster 2024-09-13T00:41:58.394656+0000 mgr.y (mgr.25304) 132 : cluster [DBG] pgmap v233: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 75 KiB/s rd, 11 KiB/s wr, 106 op/s 2024-09-13T00:41:59.243 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:58 smithi077 bash[83127]: audit 2024-09-13T00:41:58.412107+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:41:59.243 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:58 smithi077 bash[83127]: audit 2024-09-13T00:41:58.850159+0000 mon.a (mon.0) 381 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:59.244 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:58 smithi077 bash[83127]: audit 2024-09-13T00:41:58.855675+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:41:59.244 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:58 smithi077 bash[83127]: audit 2024-09-13T00:41:58.857576+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi077.bigbbf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:41:59.244 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:58 smithi077 bash[83127]: audit 2024-09-13T00:41:58.858513+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:00.011 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock (2031 ms) 2024-09-13T00:42:00.011 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeDisabled 2024-09-13T00:42:00.190 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeDisabled (178 ms) 2024-09-13T00:42:00.190 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage 2024-09-13T00:42:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:59 smithi077 bash[80681]: cephadm 2024-09-13T00:41:58.857217+0000 mgr.y (mgr.25304) 133 : cephadm [INF] Reconfiguring mds.foo.smithi077.bigbbf (monmap changed)... 2024-09-13T00:42:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:59 smithi077 bash[80681]: cephadm 2024-09-13T00:41:58.859424+0000 mgr.y (mgr.25304) 134 : cephadm [INF] Reconfiguring daemon mds.foo.smithi077.bigbbf on smithi077 2024-09-13T00:42:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:41:59 smithi077 bash[80681]: cluster 2024-09-13T00:41:58.903390+0000 mon.a (mon.0) 385 : cluster [DBG] osdmap e431: 8 total, 8 up, 8 in 2024-09-13T00:42:00.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:59 smithi077 bash[83127]: cephadm 2024-09-13T00:41:58.857217+0000 mgr.y (mgr.25304) 133 : cephadm [INF] Reconfiguring mds.foo.smithi077.bigbbf (monmap changed)... 2024-09-13T00:42:00.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:59 smithi077 bash[83127]: cephadm 2024-09-13T00:41:58.859424+0000 mgr.y (mgr.25304) 134 : cephadm [INF] Reconfiguring daemon mds.foo.smithi077.bigbbf on smithi077 2024-09-13T00:42:00.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:41:59 smithi077 bash[83127]: cluster 2024-09-13T00:41:58.903390+0000 mon.a (mon.0) 385 : cluster [DBG] osdmap e431: 8 total, 8 up, 8 in 2024-09-13T00:42:00.349 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage (160 ms) 2024-09-13T00:42:00.349 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool 2024-09-13T00:42:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:59 smithi088 bash[68960]: cephadm 2024-09-13T00:41:58.857217+0000 mgr.y (mgr.25304) 133 : cephadm [INF] Reconfiguring mds.foo.smithi077.bigbbf (monmap changed)... 2024-09-13T00:42:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:59 smithi088 bash[68960]: cephadm 2024-09-13T00:41:58.859424+0000 mgr.y (mgr.25304) 134 : cephadm [INF] Reconfiguring daemon mds.foo.smithi077.bigbbf on smithi077 2024-09-13T00:42:00.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:41:59 smithi088 bash[68960]: cluster 2024-09-13T00:41:58.903390+0000 mon.a (mon.0) 385 : cluster [DBG] osdmap e431: 8 total, 8 up, 8 in 2024-09-13T00:42:00.563 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool (215 ms) 2024-09-13T00:42:00.563 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling 2024-09-13T00:42:00.684 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling (120 ms) 2024-09-13T00:42:00.684 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling 2024-09-13T00:42:00.817 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling (134 ms) 2024-09-13T00:42:00.818 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeDisabled 2024-09-13T00:42:01.048 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeDisabled (230 ms) 2024-09-13T00:42:01.048 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage 2024-09-13T00:42:01.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:00 smithi077 bash[80681]: cluster 2024-09-13T00:41:59.915113+0000 mon.a (mon.0) 386 : cluster [DBG] osdmap e432: 8 total, 8 up, 8 in 2024-09-13T00:42:01.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:00 smithi077 bash[80681]: cluster 2024-09-13T00:42:00.395117+0000 mgr.y (mgr.25304) 135 : cluster [DBG] pgmap v236: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 350 KiB/s rd, 45 KiB/s wr, 506 op/s 2024-09-13T00:42:01.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:00 smithi077 bash[83127]: cluster 2024-09-13T00:41:59.915113+0000 mon.a (mon.0) 386 : cluster [DBG] osdmap e432: 8 total, 8 up, 8 in 2024-09-13T00:42:01.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:00 smithi077 bash[83127]: cluster 2024-09-13T00:42:00.395117+0000 mgr.y (mgr.25304) 135 : cluster [DBG] pgmap v236: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 350 KiB/s rd, 45 KiB/s wr, 506 op/s 2024-09-13T00:42:01.309 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage (262 ms) 2024-09-13T00:42:01.310 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled 2024-09-13T00:42:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:00 smithi088 bash[68960]: cluster 2024-09-13T00:41:59.915113+0000 mon.a (mon.0) 386 : cluster [DBG] osdmap e432: 8 total, 8 up, 8 in 2024-09-13T00:42:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:00 smithi088 bash[68960]: cluster 2024-09-13T00:42:00.395117+0000 mgr.y (mgr.25304) 135 : cluster [DBG] pgmap v236: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 350 KiB/s rd, 45 KiB/s wr, 506 op/s 2024-09-13T00:42:02.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:42:02 smithi088 bash[61791]: ts=2024-09-13T00:42:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:03.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[83127]: audit 2024-09-13T00:42:01.725539+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:03.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[83127]: audit 2024-09-13T00:42:01.731695+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:03.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[83127]: cephadm 2024-09-13T00:42:01.732732+0000 mgr.y (mgr.25304) 136 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-13T00:42:03.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[83127]: audit 2024-09-13T00:42:01.733097+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:42:03.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[83127]: audit 2024-09-13T00:42:01.733987+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:42:03.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[83127]: audit 2024-09-13T00:42:01.734763+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:03.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[83127]: cephadm 2024-09-13T00:42:01.735733+0000 mgr.y (mgr.25304) 137 : cephadm [INF] Reconfiguring daemon mon.b on smithi088 2024-09-13T00:42:03.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[83127]: cluster 2024-09-13T00:42:01.925166+0000 mon.a (mon.0) 392 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2024-09-13T00:42:03.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[83127]: cluster 2024-09-13T00:42:02.395758+0000 mgr.y (mgr.25304) 138 : cluster [DBG] pgmap v238: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 311 KiB/s rd, 40 KiB/s wr, 449 op/s 2024-09-13T00:42:03.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[83127]: audit 2024-09-13T00:42:02.635053+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:03.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[80681]: audit 2024-09-13T00:42:01.725539+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:03.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[80681]: audit 2024-09-13T00:42:01.731695+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:03.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[80681]: cephadm 2024-09-13T00:42:01.732732+0000 mgr.y (mgr.25304) 136 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-13T00:42:03.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[80681]: audit 2024-09-13T00:42:01.733097+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:42:03.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[80681]: audit 2024-09-13T00:42:01.733987+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:42:03.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[80681]: audit 2024-09-13T00:42:01.734763+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:03.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[80681]: cephadm 2024-09-13T00:42:01.735733+0000 mgr.y (mgr.25304) 137 : cephadm [INF] Reconfiguring daemon mon.b on smithi088 2024-09-13T00:42:03.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[80681]: cluster 2024-09-13T00:42:01.925166+0000 mon.a (mon.0) 392 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2024-09-13T00:42:03.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[80681]: cluster 2024-09-13T00:42:02.395758+0000 mgr.y (mgr.25304) 138 : cluster [DBG] pgmap v238: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 311 KiB/s rd, 40 KiB/s wr, 449 op/s 2024-09-13T00:42:03.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:02 smithi077 bash[80681]: audit 2024-09-13T00:42:02.635053+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:03.069 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled (1759 ms) 2024-09-13T00:42:03.069 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModePool 2024-09-13T00:42:03.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:02 smithi088 bash[68960]: audit 2024-09-13T00:42:01.725539+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:03.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:02 smithi088 bash[68960]: audit 2024-09-13T00:42:01.731695+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:03.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:02 smithi088 bash[68960]: cephadm 2024-09-13T00:42:01.732732+0000 mgr.y (mgr.25304) 136 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-09-13T00:42:03.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:02 smithi088 bash[68960]: audit 2024-09-13T00:42:01.733097+0000 mon.a (mon.0) 389 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-13T00:42:03.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:02 smithi088 bash[68960]: audit 2024-09-13T00:42:01.733987+0000 mon.a (mon.0) 390 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-13T00:42:03.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:02 smithi088 bash[68960]: audit 2024-09-13T00:42:01.734763+0000 mon.a (mon.0) 391 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:03.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:02 smithi088 bash[68960]: cephadm 2024-09-13T00:42:01.735733+0000 mgr.y (mgr.25304) 137 : cephadm [INF] Reconfiguring daemon mon.b on smithi088 2024-09-13T00:42:03.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:02 smithi088 bash[68960]: cluster 2024-09-13T00:42:01.925166+0000 mon.a (mon.0) 392 : cluster [DBG] osdmap e433: 8 total, 8 up, 8 in 2024-09-13T00:42:03.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:02 smithi088 bash[68960]: cluster 2024-09-13T00:42:02.395758+0000 mgr.y (mgr.25304) 138 : cluster [DBG] pgmap v238: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 311 KiB/s rd, 40 KiB/s wr, 449 op/s 2024-09-13T00:42:03.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:02 smithi088 bash[68960]: audit 2024-09-13T00:42:02.635053+0000 mon.a (mon.0) 393 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:03.317 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModePool (249 ms) 2024-09-13T00:42:03.318 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModePool 2024-09-13T00:42:03.496 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModePool (178 ms) 2024-09-13T00:42:03.496 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModeImage 2024-09-13T00:42:03.638 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModeImage (142 ms) 2024-09-13T00:42:03.638 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode 2024-09-13T00:42:03.788 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:42:03 smithi088 bash[61791]: ts=2024-09-13T00:42:03.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:03.924 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode (286 ms) 2024-09-13T00:42:03.924 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode 2024-09-13T00:42:04.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:03 smithi088 bash[68960]: cluster 2024-09-13T00:42:02.932433+0000 mon.a (mon.0) 394 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2024-09-13T00:42:04.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:03 smithi077 bash[80681]: cluster 2024-09-13T00:42:02.932433+0000 mon.a (mon.0) 394 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2024-09-13T00:42:04.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:03 smithi077 bash[83127]: cluster 2024-09-13T00:42:02.932433+0000 mon.a (mon.0) 394 : cluster [DBG] osdmap e434: 8 total, 8 up, 8 in 2024-09-13T00:42:04.313 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode (390 ms) 2024-09-13T00:42:04.313 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode 2024-09-13T00:42:04.666 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode (353 ms) 2024-09-13T00:42:04.666 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode 2024-09-13T00:42:04.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:42:04 smithi088 bash[61791]: ts=2024-09-13T00:42:04.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:04.852 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:42:04 smithi088 bash[61791]: ts=2024-09-13T00:42:04.455Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:05.002 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode (336 ms) 2024-09-13T00:42:05.003 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode 2024-09-13T00:42:05.331 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode (328 ms) 2024-09-13T00:42:05.331 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode 2024-09-13T00:42:05.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:05 smithi088 bash[68960]: audit 2024-09-13T00:42:04.315965+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:05.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:05 smithi088 bash[68960]: audit 2024-09-13T00:42:04.326950+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:05.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:05 smithi088 bash[68960]: cephadm 2024-09-13T00:42:04.329099+0000 mgr.y (mgr.25304) 139 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-09-13T00:42:05.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:05 smithi088 bash[68960]: audit 2024-09-13T00:42:04.329790+0000 mon.a (mon.0) 397 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:42:05.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:05 smithi088 bash[68960]: audit 2024-09-13T00:42:04.331981+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:42:05.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:05 smithi088 bash[68960]: audit 2024-09-13T00:42:04.334008+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:05.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:05 smithi088 bash[68960]: cephadm 2024-09-13T00:42:04.336347+0000 mgr.y (mgr.25304) 140 : cephadm [INF] Reconfiguring daemon mgr.x on smithi088 2024-09-13T00:42:05.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:05 smithi088 bash[68960]: cluster 2024-09-13T00:42:04.397162+0000 mgr.y (mgr.25304) 141 : cluster [DBG] pgmap v240: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 702 KiB/s rd, 89 KiB/s wr, 1.01k op/s 2024-09-13T00:42:05.706 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode (375 ms) 2024-09-13T00:42:05.706 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageEnabled 2024-09-13T00:42:05.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:05 smithi077 bash[83127]: audit 2024-09-13T00:42:04.315965+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:05.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:05 smithi077 bash[83127]: audit 2024-09-13T00:42:04.326950+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:05.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:05 smithi077 bash[83127]: cephadm 2024-09-13T00:42:04.329099+0000 mgr.y (mgr.25304) 139 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-09-13T00:42:05.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:05 smithi077 bash[83127]: audit 2024-09-13T00:42:04.329790+0000 mon.a (mon.0) 397 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:42:05.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:05 smithi077 bash[83127]: audit 2024-09-13T00:42:04.331981+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:42:05.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:05 smithi077 bash[83127]: audit 2024-09-13T00:42:04.334008+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:05.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:05 smithi077 bash[83127]: cephadm 2024-09-13T00:42:04.336347+0000 mgr.y (mgr.25304) 140 : cephadm [INF] Reconfiguring daemon mgr.x on smithi088 2024-09-13T00:42:05.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:05 smithi077 bash[83127]: cluster 2024-09-13T00:42:04.397162+0000 mgr.y (mgr.25304) 141 : cluster [DBG] pgmap v240: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 702 KiB/s rd, 89 KiB/s wr, 1.01k op/s 2024-09-13T00:42:05.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:05 smithi077 bash[80681]: audit 2024-09-13T00:42:04.315965+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:05.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:05 smithi077 bash[80681]: audit 2024-09-13T00:42:04.326950+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:05.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:05 smithi077 bash[80681]: cephadm 2024-09-13T00:42:04.329099+0000 mgr.y (mgr.25304) 139 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-09-13T00:42:05.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:05 smithi077 bash[80681]: audit 2024-09-13T00:42:04.329790+0000 mon.a (mon.0) 397 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-13T00:42:05.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:05 smithi077 bash[80681]: audit 2024-09-13T00:42:04.331981+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-13T00:42:05.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:05 smithi077 bash[80681]: audit 2024-09-13T00:42:04.334008+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:05.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:05 smithi077 bash[80681]: cephadm 2024-09-13T00:42:04.336347+0000 mgr.y (mgr.25304) 140 : cephadm [INF] Reconfiguring daemon mgr.x on smithi088 2024-09-13T00:42:05.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:05 smithi077 bash[80681]: cluster 2024-09-13T00:42:04.397162+0000 mgr.y (mgr.25304) 141 : cluster [DBG] pgmap v240: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 702 KiB/s rd, 89 KiB/s wr, 1.01k op/s 2024-09-13T00:42:05.878 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageEnabled (172 ms) 2024-09-13T00:42:05.878 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDisabled 2024-09-13T00:42:06.055 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDisabled (177 ms) 2024-09-13T00:42:06.055 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.RemoveImage_With_ImageWithoutJournal 2024-09-13T00:42:06.221 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.RemoveImage_With_ImageWithoutJournal (166 ms) 2024-09-13T00:42:06.221 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDemoted 2024-09-13T00:42:06.451 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDemoted (231 ms) 2024-09-13T00:42:06.452 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.TrashMoveRestore_PoolMode 2024-09-13T00:42:06.713 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.TrashMoveRestore_PoolMode (261 ms) 2024-09-13T00:42:06.713 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled 2024-09-13T00:42:06.967 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled (255 ms) 2024-09-13T00:42:06.967 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled 2024-09-13T00:42:07.213 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled (245 ms) 2024-09-13T00:42:07.213 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.MirrorStatusList 2024-09-13T00:42:07.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:06 smithi077 bash[83127]: cluster 2024-09-13T00:42:05.876459+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2024-09-13T00:42:07.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:06 smithi077 bash[83127]: cluster 2024-09-13T00:42:06.397687+0000 mgr.y (mgr.25304) 142 : cluster [DBG] pgmap v242: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 409 KiB/s rd, 51 KiB/s wr, 591 op/s 2024-09-13T00:42:07.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:06 smithi077 bash[83127]: audit 2024-09-13T00:42:06.504930+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:07.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:06 smithi077 bash[83127]: audit 2024-09-13T00:42:06.514263+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:07.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:06 smithi077 bash[83127]: audit 2024-09-13T00:42:06.515993+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T00:42:07.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:06 smithi077 bash[83127]: audit 2024-09-13T00:42:06.516977+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:07.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:06 smithi077 bash[80681]: cluster 2024-09-13T00:42:05.876459+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2024-09-13T00:42:07.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:06 smithi077 bash[80681]: cluster 2024-09-13T00:42:06.397687+0000 mgr.y (mgr.25304) 142 : cluster [DBG] pgmap v242: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 409 KiB/s rd, 51 KiB/s wr, 591 op/s 2024-09-13T00:42:07.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:06 smithi077 bash[80681]: audit 2024-09-13T00:42:06.504930+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:07.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:06 smithi077 bash[80681]: audit 2024-09-13T00:42:06.514263+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:07.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:06 smithi077 bash[80681]: audit 2024-09-13T00:42:06.515993+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T00:42:07.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:06 smithi077 bash[80681]: audit 2024-09-13T00:42:06.516977+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:07.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:06 smithi088 bash[68960]: cluster 2024-09-13T00:42:05.876459+0000 mon.a (mon.0) 400 : cluster [DBG] osdmap e435: 8 total, 8 up, 8 in 2024-09-13T00:42:07.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:06 smithi088 bash[68960]: cluster 2024-09-13T00:42:06.397687+0000 mgr.y (mgr.25304) 142 : cluster [DBG] pgmap v242: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 409 KiB/s rd, 51 KiB/s wr, 591 op/s 2024-09-13T00:42:07.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:06 smithi088 bash[68960]: audit 2024-09-13T00:42:06.504930+0000 mon.a (mon.0) 401 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:07.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:06 smithi088 bash[68960]: audit 2024-09-13T00:42:06.514263+0000 mon.a (mon.0) 402 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:07.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:06 smithi088 bash[68960]: audit 2024-09-13T00:42:06.515993+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T00:42:07.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:06 smithi088 bash[68960]: audit 2024-09-13T00:42:06.516977+0000 mon.a (mon.0) 404 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:07.358 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.MirrorStatusList (146 ms) 2024-09-13T00:42:07.359 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.RemoveBootstrapped 2024-09-13T00:42:07.582 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.RemoveBootstrapped (223 ms) 2024-09-13T00:42:07.582 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.AioPromoteDemote 2024-09-13T00:42:08.178 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.AioPromoteDemote (597 ms) 2024-09-13T00:42:08.178 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.AioGetInfo 2024-09-13T00:42:08.229 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:07 smithi088 bash[68960]: cephadm 2024-09-13T00:42:06.515657+0000 mgr.y (mgr.25304) 143 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-09-13T00:42:08.230 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:07 smithi088 bash[68960]: cephadm 2024-09-13T00:42:06.519628+0000 mgr.y (mgr.25304) 144 : cephadm [INF] Reconfiguring daemon osd.4 on smithi088 2024-09-13T00:42:08.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:07 smithi077 bash[83127]: cephadm 2024-09-13T00:42:06.515657+0000 mgr.y (mgr.25304) 143 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-09-13T00:42:08.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:07 smithi077 bash[83127]: cephadm 2024-09-13T00:42:06.519628+0000 mgr.y (mgr.25304) 144 : cephadm [INF] Reconfiguring daemon osd.4 on smithi088 2024-09-13T00:42:08.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:07 smithi077 bash[80681]: cephadm 2024-09-13T00:42:06.515657+0000 mgr.y (mgr.25304) 143 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-09-13T00:42:08.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:07 smithi077 bash[80681]: cephadm 2024-09-13T00:42:06.519628+0000 mgr.y (mgr.25304) 144 : cephadm [INF] Reconfiguring daemon osd.4 on smithi088 2024-09-13T00:42:08.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:42:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:42:08] "GET /metrics HTTP/1.1" 200 36419 "" "Prometheus/2.43.0" 2024-09-13T00:42:08.828 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.AioGetInfo (650 ms) 2024-09-13T00:42:08.829 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.AioGetStatus 2024-09-13T00:42:09.101 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:42:08 smithi088 bash[61791]: ts=2024-09-13T00:42:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:09.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:08 smithi088 bash[68960]: cluster 2024-09-13T00:42:08.398485+0000 mgr.y (mgr.25304) 145 : cluster [DBG] pgmap v243: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 379 KiB/s rd, 47 KiB/s wr, 547 op/s 2024-09-13T00:42:09.314 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:08 smithi088 bash[68960]: audit 2024-09-13T00:42:08.849719+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:09.314 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:08 smithi088 bash[68960]: audit 2024-09-13T00:42:08.856158+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:09.314 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:08 smithi088 bash[68960]: audit 2024-09-13T00:42:08.857551+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T00:42:09.314 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:08 smithi088 bash[68960]: audit 2024-09-13T00:42:08.858307+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:09.315 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.AioGetStatus (355 ms) 2024-09-13T00:42:09.315 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.SiteName 2024-09-13T00:42:09.315 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.SiteName (11 ms) 2024-09-13T00:42:09.315 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.Bootstrap 2024-09-13T00:42:09.316 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:08 smithi077 bash[80681]: cluster 2024-09-13T00:42:08.398485+0000 mgr.y (mgr.25304) 145 : cluster [DBG] pgmap v243: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 379 KiB/s rd, 47 KiB/s wr, 547 op/s 2024-09-13T00:42:09.316 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:08 smithi077 bash[80681]: audit 2024-09-13T00:42:08.849719+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:09.316 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:08 smithi077 bash[80681]: audit 2024-09-13T00:42:08.856158+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:09.316 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:08 smithi077 bash[80681]: audit 2024-09-13T00:42:08.857551+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T00:42:09.316 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:08 smithi077 bash[80681]: audit 2024-09-13T00:42:08.858307+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:09.316 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:08 smithi077 bash[83127]: cluster 2024-09-13T00:42:08.398485+0000 mgr.y (mgr.25304) 145 : cluster [DBG] pgmap v243: 105 pgs: 105 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 379 KiB/s rd, 47 KiB/s wr, 547 op/s 2024-09-13T00:42:09.317 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:08 smithi077 bash[83127]: audit 2024-09-13T00:42:08.849719+0000 mon.a (mon.0) 405 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:09.317 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:08 smithi077 bash[83127]: audit 2024-09-13T00:42:08.856158+0000 mon.a (mon.0) 406 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:09.317 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:08 smithi077 bash[83127]: audit 2024-09-13T00:42:08.857551+0000 mon.a (mon.0) 407 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-13T00:42:09.317 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:08 smithi077 bash[83127]: audit 2024-09-13T00:42:08.858307+0000 mon.a (mon.0) 408 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:10.516 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.Bootstrap (1321 ms) 2024-09-13T00:42:10.516 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.PeerDirection 2024-09-13T00:42:10.527 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.PeerDirection (12 ms) 2024-09-13T00:42:10.528 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.Snapshot 2024-09-13T00:42:10.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:10 smithi077 bash[80681]: cephadm 2024-09-13T00:42:08.857252+0000 mgr.y (mgr.25304) 146 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-09-13T00:42:10.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:10 smithi077 bash[80681]: cephadm 2024-09-13T00:42:08.860530+0000 mgr.y (mgr.25304) 147 : cephadm [INF] Reconfiguring daemon osd.5 on smithi088 2024-09-13T00:42:10.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:10 smithi077 bash[80681]: audit 2024-09-13T00:42:09.191310+0000 mon.a (mon.0) 409 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:42:10.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:10 smithi077 bash[80681]: audit 2024-09-13T00:42:09.191957+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:42:10.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:10 smithi077 bash[80681]: audit 2024-09-13T00:42:09.192316+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:42:10.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:10 smithi077 bash[80681]: audit 2024-09-13T00:42:09.192607+0000 mon.a (mon.0) 410 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:42:10.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:10 smithi077 bash[80681]: audit 2024-09-13T00:42:09.197929+0000 mon.a (mon.0) 411 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-13T00:42:10.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:10 smithi077 bash[80681]: audit 2024-09-13T00:42:09.198644+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:42:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:10 smithi077 bash[83127]: cephadm 2024-09-13T00:42:08.857252+0000 mgr.y (mgr.25304) 146 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-09-13T00:42:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:10 smithi077 bash[83127]: cephadm 2024-09-13T00:42:08.860530+0000 mgr.y (mgr.25304) 147 : cephadm [INF] Reconfiguring daemon osd.5 on smithi088 2024-09-13T00:42:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:10 smithi077 bash[83127]: audit 2024-09-13T00:42:09.191310+0000 mon.a (mon.0) 409 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:42:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:10 smithi077 bash[83127]: audit 2024-09-13T00:42:09.191957+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:42:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:10 smithi077 bash[83127]: audit 2024-09-13T00:42:09.192316+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:42:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:10 smithi077 bash[83127]: audit 2024-09-13T00:42:09.192607+0000 mon.a (mon.0) 410 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:42:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:10 smithi077 bash[83127]: audit 2024-09-13T00:42:09.197929+0000 mon.a (mon.0) 411 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-13T00:42:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:10 smithi077 bash[83127]: audit 2024-09-13T00:42:09.198644+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:42:10.536 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:10 smithi088 bash[68960]: cephadm 2024-09-13T00:42:08.857252+0000 mgr.y (mgr.25304) 146 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-09-13T00:42:10.536 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:10 smithi088 bash[68960]: cephadm 2024-09-13T00:42:08.860530+0000 mgr.y (mgr.25304) 147 : cephadm [INF] Reconfiguring daemon osd.5 on smithi088 2024-09-13T00:42:10.537 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:10 smithi088 bash[68960]: audit 2024-09-13T00:42:09.191310+0000 mon.a (mon.0) 409 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:42:10.537 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:10 smithi088 bash[68960]: audit 2024-09-13T00:42:09.191957+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:42:10.537 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:10 smithi088 bash[68960]: audit 2024-09-13T00:42:09.192316+0000 mon.c (mon.2) 5 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:42:10.537 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:10 smithi088 bash[68960]: audit 2024-09-13T00:42:09.192607+0000 mon.a (mon.0) 410 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:42:10.537 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:10 smithi088 bash[68960]: audit 2024-09-13T00:42:09.197929+0000 mon.a (mon.0) 411 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-13T00:42:10.537 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:10 smithi088 bash[68960]: audit 2024-09-13T00:42:09.198644+0000 mon.c (mon.2) 6 : audit [DBG] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:42:11.456 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:11 smithi088 bash[68960]: cluster 2024-09-13T00:42:10.399307+0000 mgr.y (mgr.25304) 148 : cluster [DBG] pgmap v244: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 161 KiB/s wr, 1.43k op/s 2024-09-13T00:42:11.456 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:11 smithi088 bash[68960]: audit 2024-09-13T00:42:10.496520+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-09-13T00:42:11.456 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:11 smithi088 bash[68960]: audit 2024-09-13T00:42:10.502167+0000 mon.a (mon.0) 412 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:42:11.457 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:11 smithi088 bash[68960]: audit 2024-09-13T00:42:10.502850+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:42:11.457 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:11 smithi088 bash[68960]: audit 2024-09-13T00:42:10.503177+0000 mon.a (mon.0) 413 : audit [INF] from='client.? ' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:42:11.457 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:11 smithi088 bash[68960]: audit 2024-09-13T00:42:10.507883+0000 mon.a (mon.0) 414 : audit [INF] from='client.? ' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-09-13T00:42:11.457 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:11 smithi088 bash[68960]: audit 2024-09-13T00:42:10.508547+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:42:11.457 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:11 smithi088 bash[68960]: audit 2024-09-13T00:42:10.524649+0000 mon.c (mon.2) 10 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d31bd7a2-09a4-4977-bece-f2984a40682d"}]: dispatch 2024-09-13T00:42:11.457 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:11 smithi088 bash[68960]: audit 2024-09-13T00:42:10.524960+0000 mon.a (mon.0) 415 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d31bd7a2-09a4-4977-bece-f2984a40682d"}]: dispatch 2024-09-13T00:42:11.457 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:11 smithi088 bash[68960]: audit 2024-09-13T00:42:10.528525+0000 mon.a (mon.0) 416 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d31bd7a2-09a4-4977-bece-f2984a40682d"}]': finished 2024-09-13T00:42:11.458 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:11 smithi088 bash[68960]: audit 2024-09-13T00:42:11.067460+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:11.458 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:11 smithi088 bash[68960]: audit 2024-09-13T00:42:11.077790+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:11.458 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:11 smithi088 bash[68960]: audit 2024-09-13T00:42:11.080355+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T00:42:11.458 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:11 smithi088 bash[68960]: audit 2024-09-13T00:42:11.081979+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:11.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[80681]: cluster 2024-09-13T00:42:10.399307+0000 mgr.y (mgr.25304) 148 : cluster [DBG] pgmap v244: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 161 KiB/s wr, 1.43k op/s 2024-09-13T00:42:11.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[80681]: audit 2024-09-13T00:42:10.496520+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-09-13T00:42:11.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[80681]: audit 2024-09-13T00:42:10.502167+0000 mon.a (mon.0) 412 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:42:11.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[80681]: audit 2024-09-13T00:42:10.502850+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:42:11.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[80681]: audit 2024-09-13T00:42:10.503177+0000 mon.a (mon.0) 413 : audit [INF] from='client.? ' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:42:11.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[80681]: audit 2024-09-13T00:42:10.507883+0000 mon.a (mon.0) 414 : audit [INF] from='client.? ' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-09-13T00:42:11.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[80681]: audit 2024-09-13T00:42:10.508547+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:42:11.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[80681]: audit 2024-09-13T00:42:10.524649+0000 mon.c (mon.2) 10 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d31bd7a2-09a4-4977-bece-f2984a40682d"}]: dispatch 2024-09-13T00:42:11.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[80681]: audit 2024-09-13T00:42:10.524960+0000 mon.a (mon.0) 415 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d31bd7a2-09a4-4977-bece-f2984a40682d"}]: dispatch 2024-09-13T00:42:11.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[80681]: audit 2024-09-13T00:42:10.528525+0000 mon.a (mon.0) 416 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d31bd7a2-09a4-4977-bece-f2984a40682d"}]': finished 2024-09-13T00:42:11.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[80681]: audit 2024-09-13T00:42:11.067460+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:11.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[80681]: audit 2024-09-13T00:42:11.077790+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:11.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[80681]: audit 2024-09-13T00:42:11.080355+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T00:42:11.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[80681]: audit 2024-09-13T00:42:11.081979+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:11.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[83127]: cluster 2024-09-13T00:42:10.399307+0000 mgr.y (mgr.25304) 148 : cluster [DBG] pgmap v244: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.0 MiB/s rd, 161 KiB/s wr, 1.43k op/s 2024-09-13T00:42:11.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[83127]: audit 2024-09-13T00:42:10.496520+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-09-13T00:42:11.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[83127]: audit 2024-09-13T00:42:10.502167+0000 mon.a (mon.0) 412 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:42:11.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[83127]: audit 2024-09-13T00:42:10.502850+0000 mon.c (mon.2) 8 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:42:11.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[83127]: audit 2024-09-13T00:42:10.503177+0000 mon.a (mon.0) 413 : audit [INF] from='client.? ' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-09-13T00:42:11.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[83127]: audit 2024-09-13T00:42:10.507883+0000 mon.a (mon.0) 414 : audit [INF] from='client.? ' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-09-13T00:42:11.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[83127]: audit 2024-09-13T00:42:10.508547+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-09-13T00:42:11.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[83127]: audit 2024-09-13T00:42:10.524649+0000 mon.c (mon.2) 10 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d31bd7a2-09a4-4977-bece-f2984a40682d"}]: dispatch 2024-09-13T00:42:11.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[83127]: audit 2024-09-13T00:42:10.524960+0000 mon.a (mon.0) 415 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d31bd7a2-09a4-4977-bece-f2984a40682d"}]: dispatch 2024-09-13T00:42:11.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[83127]: audit 2024-09-13T00:42:10.528525+0000 mon.a (mon.0) 416 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d31bd7a2-09a4-4977-bece-f2984a40682d"}]': finished 2024-09-13T00:42:11.531 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[83127]: audit 2024-09-13T00:42:11.067460+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:11.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[83127]: audit 2024-09-13T00:42:11.077790+0000 mon.a (mon.0) 418 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:11.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[83127]: audit 2024-09-13T00:42:11.080355+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-13T00:42:11.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:11 smithi077 bash[83127]: audit 2024-09-13T00:42:11.081979+0000 mon.a (mon.0) 420 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:12.456 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:12 smithi088 bash[68960]: cephadm 2024-09-13T00:42:11.079763+0000 mgr.y (mgr.25304) 149 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-09-13T00:42:12.457 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:12 smithi088 bash[68960]: cephadm 2024-09-13T00:42:11.086924+0000 mgr.y (mgr.25304) 150 : cephadm [INF] Reconfiguring daemon osd.6 on smithi088 2024-09-13T00:42:12.457 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:12 smithi088 bash[68960]: cluster 2024-09-13T00:42:11.538890+0000 mon.a (mon.0) 421 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2024-09-13T00:42:12.457 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:42:12 smithi088 bash[61791]: ts=2024-09-13T00:42:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:12.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:12 smithi077 bash[80681]: cephadm 2024-09-13T00:42:11.079763+0000 mgr.y (mgr.25304) 149 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-09-13T00:42:12.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:12 smithi077 bash[80681]: cephadm 2024-09-13T00:42:11.086924+0000 mgr.y (mgr.25304) 150 : cephadm [INF] Reconfiguring daemon osd.6 on smithi088 2024-09-13T00:42:12.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:12 smithi077 bash[80681]: cluster 2024-09-13T00:42:11.538890+0000 mon.a (mon.0) 421 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2024-09-13T00:42:12.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:12 smithi077 bash[83127]: cephadm 2024-09-13T00:42:11.079763+0000 mgr.y (mgr.25304) 149 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-09-13T00:42:12.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:12 smithi077 bash[83127]: cephadm 2024-09-13T00:42:11.086924+0000 mgr.y (mgr.25304) 150 : cephadm [INF] Reconfiguring daemon osd.6 on smithi088 2024-09-13T00:42:12.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:12 smithi077 bash[83127]: cluster 2024-09-13T00:42:11.538890+0000 mon.a (mon.0) 421 : cluster [DBG] osdmap e436: 8 total, 8 up, 8 in 2024-09-13T00:42:13.505 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:13 smithi088 bash[68960]: cluster 2024-09-13T00:42:12.400058+0000 mgr.y (mgr.25304) 151 : cluster [DBG] pgmap v246: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 730 KiB/s rd, 123 KiB/s wr, 990 op/s 2024-09-13T00:42:13.506 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:13 smithi088 bash[68960]: cluster 2024-09-13T00:42:12.540090+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2024-09-13T00:42:13.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:13 smithi077 bash[80681]: cluster 2024-09-13T00:42:12.400058+0000 mgr.y (mgr.25304) 151 : cluster [DBG] pgmap v246: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 730 KiB/s rd, 123 KiB/s wr, 990 op/s 2024-09-13T00:42:13.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:13 smithi077 bash[80681]: cluster 2024-09-13T00:42:12.540090+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2024-09-13T00:42:13.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:13 smithi077 bash[83127]: cluster 2024-09-13T00:42:12.400058+0000 mgr.y (mgr.25304) 151 : cluster [DBG] pgmap v246: 105 pgs: 105 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 730 KiB/s rd, 123 KiB/s wr, 990 op/s 2024-09-13T00:42:13.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:13 smithi077 bash[83127]: cluster 2024-09-13T00:42:12.540090+0000 mon.a (mon.0) 422 : cluster [DBG] osdmap e437: 8 total, 8 up, 8 in 2024-09-13T00:42:13.851 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:42:13 smithi088 bash[61791]: ts=2024-09-13T00:42:13.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi077\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", domainname=\"(none)\", instance=\"smithi077\", job=\"node\", machine=\"x86_64\", nodename=\"smithi077\", release=\"5.15.0-121-generic\", sysname=\"Linux\", version=\"#131-Ubuntu SMP Fri Aug 9 08:29:53 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:14.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:42:14 smithi088 bash[61791]: ts=2024-09-13T00:42:14.454Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsInactive index=0 msg="Evaluating rule failed" rule="alert: CephPGsInactive\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_active)\n > 0\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.1\n severity: critical\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been inactive for more than 5 minutes in pool\n {{ $labels.name }}. Inactive placement groups are not able to serve read/write\n requests.'\n summary: One or more placement groups are inactive\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:14.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:42:14 smithi088 bash[61791]: ts=2024-09-13T00:42:14.456Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pgs name=CephPGsUnclean index=1 msg="Evaluating rule failed" rule="alert: CephPGsUnclean\nexpr: ceph_pool_metadata * on (pool_id, instance) group_left () (ceph_pg_total - ceph_pg_clean)\n > 0\nfor: 15m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.7.2\n severity: warning\n type: ceph_default\nannotations:\n description: '{{ $value }} PGs have been unclean for more than 15 minutes in pool\n {{ $labels.name }}. Unclean PGs have not recovered from a previous failure.'\n summary: One or more placement groups are marked unclean\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the right hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:14 smithi088 bash[68960]: audit 2024-09-13T00:42:13.291566+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:14 smithi088 bash[68960]: audit 2024-09-13T00:42:13.303781+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:14 smithi088 bash[68960]: cephadm 2024-09-13T00:42:13.305694+0000 mgr.y (mgr.25304) 152 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-09-13T00:42:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:14 smithi088 bash[68960]: audit 2024-09-13T00:42:13.306228+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T00:42:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:14 smithi088 bash[68960]: audit 2024-09-13T00:42:13.307840+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:14 smithi088 bash[68960]: cephadm 2024-09-13T00:42:13.312668+0000 mgr.y (mgr.25304) 153 : cephadm [INF] Reconfiguring daemon osd.7 on smithi088 2024-09-13T00:42:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:14 smithi088 bash[68960]: audit 2024-09-13T00:42:13.412483+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:14 smithi088 bash[68960]: cluster 2024-09-13T00:42:13.544010+0000 mon.a (mon.0) 428 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2024-09-13T00:42:14.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:14 smithi077 bash[83127]: audit 2024-09-13T00:42:13.291566+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:14.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:14 smithi077 bash[83127]: audit 2024-09-13T00:42:13.303781+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:14.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:14 smithi077 bash[83127]: cephadm 2024-09-13T00:42:13.305694+0000 mgr.y (mgr.25304) 152 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-09-13T00:42:14.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:14 smithi077 bash[83127]: audit 2024-09-13T00:42:13.306228+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T00:42:14.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:14 smithi077 bash[83127]: audit 2024-09-13T00:42:13.307840+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:14.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:14 smithi077 bash[83127]: cephadm 2024-09-13T00:42:13.312668+0000 mgr.y (mgr.25304) 153 : cephadm [INF] Reconfiguring daemon osd.7 on smithi088 2024-09-13T00:42:14.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:14 smithi077 bash[83127]: audit 2024-09-13T00:42:13.412483+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:14.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:14 smithi077 bash[83127]: cluster 2024-09-13T00:42:13.544010+0000 mon.a (mon.0) 428 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2024-09-13T00:42:14.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:14 smithi077 bash[80681]: audit 2024-09-13T00:42:13.291566+0000 mon.a (mon.0) 423 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:14.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:14 smithi077 bash[80681]: audit 2024-09-13T00:42:13.303781+0000 mon.a (mon.0) 424 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:14.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:14 smithi077 bash[80681]: cephadm 2024-09-13T00:42:13.305694+0000 mgr.y (mgr.25304) 152 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-09-13T00:42:14.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:14 smithi077 bash[80681]: audit 2024-09-13T00:42:13.306228+0000 mon.a (mon.0) 425 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-13T00:42:14.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:14 smithi077 bash[80681]: audit 2024-09-13T00:42:13.307840+0000 mon.a (mon.0) 426 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:14.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:14 smithi077 bash[80681]: cephadm 2024-09-13T00:42:13.312668+0000 mgr.y (mgr.25304) 153 : cephadm [INF] Reconfiguring daemon osd.7 on smithi088 2024-09-13T00:42:14.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:14 smithi077 bash[80681]: audit 2024-09-13T00:42:13.412483+0000 mon.a (mon.0) 427 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:14.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:14 smithi077 bash[80681]: cluster 2024-09-13T00:42:13.544010+0000 mon.a (mon.0) 428 : cluster [DBG] osdmap e438: 8 total, 8 up, 8 in 2024-09-13T00:42:15.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:15 smithi088 bash[68960]: cluster 2024-09-13T00:42:14.401343+0000 mgr.y (mgr.25304) 154 : cluster [DBG] pgmap v249: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 184 KiB/s wr, 1.64k op/s 2024-09-13T00:42:15.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:15 smithi088 bash[68960]: cluster 2024-09-13T00:42:14.544565+0000 mon.a (mon.0) 429 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-09-13T00:42:15.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:15 smithi077 bash[80681]: cluster 2024-09-13T00:42:14.401343+0000 mgr.y (mgr.25304) 154 : cluster [DBG] pgmap v249: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 184 KiB/s wr, 1.64k op/s 2024-09-13T00:42:15.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:15 smithi077 bash[80681]: cluster 2024-09-13T00:42:14.544565+0000 mon.a (mon.0) 429 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-09-13T00:42:15.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:15 smithi077 bash[83127]: cluster 2024-09-13T00:42:14.401343+0000 mgr.y (mgr.25304) 154 : cluster [DBG] pgmap v249: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 184 KiB/s wr, 1.64k op/s 2024-09-13T00:42:15.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:15 smithi077 bash[83127]: cluster 2024-09-13T00:42:14.544565+0000 mon.a (mon.0) 429 : cluster [DBG] osdmap e439: 8 total, 8 up, 8 in 2024-09-13T00:42:16.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[80681]: audit 2024-09-13T00:42:15.517276+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:16.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[80681]: audit 2024-09-13T00:42:15.527582+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:16.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[80681]: cephadm 2024-09-13T00:42:15.530872+0000 mgr.y (mgr.25304) 155 : cephadm [INF] Reconfiguring mds.foo.smithi088.krmowh (monmap changed)... 2024-09-13T00:42:16.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[80681]: audit 2024-09-13T00:42:15.531517+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi088.krmowh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:42:16.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[80681]: audit 2024-09-13T00:42:15.533701+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:16.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[80681]: cephadm 2024-09-13T00:42:15.535745+0000 mgr.y (mgr.25304) 156 : cephadm [INF] Reconfiguring daemon mds.foo.smithi088.krmowh on smithi088 2024-09-13T00:42:16.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[80681]: cluster 2024-09-13T00:42:15.548022+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2024-09-13T00:42:16.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[80681]: cluster 2024-09-13T00:42:15.878241+0000 mon.a (mon.0) 435 : cluster [DBG] osdmap e441: 8 total, 8 up, 8 in 2024-09-13T00:42:16.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[80681]: cluster 2024-09-13T00:42:16.402034+0000 mgr.y (mgr.25304) 157 : cluster [DBG] pgmap v253: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 369 KiB/s rd, 31 KiB/s wr, 501 op/s 2024-09-13T00:42:16.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[83127]: audit 2024-09-13T00:42:15.517276+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:16.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[83127]: audit 2024-09-13T00:42:15.527582+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:16.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[83127]: cephadm 2024-09-13T00:42:15.530872+0000 mgr.y (mgr.25304) 155 : cephadm [INF] Reconfiguring mds.foo.smithi088.krmowh (monmap changed)... 2024-09-13T00:42:16.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[83127]: audit 2024-09-13T00:42:15.531517+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi088.krmowh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:42:16.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[83127]: audit 2024-09-13T00:42:15.533701+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:16.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[83127]: cephadm 2024-09-13T00:42:15.535745+0000 mgr.y (mgr.25304) 156 : cephadm [INF] Reconfiguring daemon mds.foo.smithi088.krmowh on smithi088 2024-09-13T00:42:16.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[83127]: cluster 2024-09-13T00:42:15.548022+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2024-09-13T00:42:16.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[83127]: cluster 2024-09-13T00:42:15.878241+0000 mon.a (mon.0) 435 : cluster [DBG] osdmap e441: 8 total, 8 up, 8 in 2024-09-13T00:42:16.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:16 smithi077 bash[83127]: cluster 2024-09-13T00:42:16.402034+0000 mgr.y (mgr.25304) 157 : cluster [DBG] pgmap v253: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 369 KiB/s rd, 31 KiB/s wr, 501 op/s 2024-09-13T00:42:16.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:16 smithi088 bash[68960]: audit 2024-09-13T00:42:15.517276+0000 mon.a (mon.0) 430 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:16.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:16 smithi088 bash[68960]: audit 2024-09-13T00:42:15.527582+0000 mon.a (mon.0) 431 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:16.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:16 smithi088 bash[68960]: cephadm 2024-09-13T00:42:15.530872+0000 mgr.y (mgr.25304) 155 : cephadm [INF] Reconfiguring mds.foo.smithi088.krmowh (monmap changed)... 2024-09-13T00:42:16.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:16 smithi088 bash[68960]: audit 2024-09-13T00:42:15.531517+0000 mon.a (mon.0) 432 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi088.krmowh", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-13T00:42:16.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:16 smithi088 bash[68960]: audit 2024-09-13T00:42:15.533701+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:16.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:16 smithi088 bash[68960]: cephadm 2024-09-13T00:42:15.535745+0000 mgr.y (mgr.25304) 156 : cephadm [INF] Reconfiguring daemon mds.foo.smithi088.krmowh on smithi088 2024-09-13T00:42:16.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:16 smithi088 bash[68960]: cluster 2024-09-13T00:42:15.548022+0000 mon.a (mon.0) 434 : cluster [DBG] osdmap e440: 8 total, 8 up, 8 in 2024-09-13T00:42:16.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:16 smithi088 bash[68960]: cluster 2024-09-13T00:42:15.878241+0000 mon.a (mon.0) 435 : cluster [DBG] osdmap e441: 8 total, 8 up, 8 in 2024-09-13T00:42:16.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:16 smithi088 bash[68960]: cluster 2024-09-13T00:42:16.402034+0000 mgr.y (mgr.25304) 157 : cluster [DBG] pgmap v253: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 369 KiB/s rd, 31 KiB/s wr, 501 op/s 2024-09-13T00:42:18.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:17 smithi077 bash[80681]: cluster 2024-09-13T00:42:16.886213+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e442: 8 total, 8 up, 8 in 2024-09-13T00:42:18.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:17 smithi077 bash[80681]: audit 2024-09-13T00:42:17.698341+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:18.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:17 smithi077 bash[80681]: audit 2024-09-13T00:42:17.710520+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:18.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:17 smithi077 bash[80681]: audit 2024-09-13T00:42:17.799628+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:42:18.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:17 smithi077 bash[83127]: cluster 2024-09-13T00:42:16.886213+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e442: 8 total, 8 up, 8 in 2024-09-13T00:42:18.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:17 smithi077 bash[83127]: audit 2024-09-13T00:42:17.698341+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:18.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:17 smithi077 bash[83127]: audit 2024-09-13T00:42:17.710520+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:18.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:17 smithi077 bash[83127]: audit 2024-09-13T00:42:17.799628+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:42:18.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:17 smithi088 bash[68960]: cluster 2024-09-13T00:42:16.886213+0000 mon.a (mon.0) 436 : cluster [DBG] osdmap e442: 8 total, 8 up, 8 in 2024-09-13T00:42:18.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:17 smithi088 bash[68960]: audit 2024-09-13T00:42:17.698341+0000 mon.a (mon.0) 437 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:18.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:17 smithi088 bash[68960]: audit 2024-09-13T00:42:17.710520+0000 mon.a (mon.0) 438 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:18.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:17 smithi088 bash[68960]: audit 2024-09-13T00:42:17.799628+0000 mon.a (mon.0) 439 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:42:18.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:42:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:42:18] "GET /metrics HTTP/1.1" 200 36436 "" "Prometheus/2.43.0" 2024-09-13T00:42:19.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:18 smithi088 bash[68960]: cluster 2024-09-13T00:42:17.889111+0000 mon.a (mon.0) 440 : cluster [DBG] osdmap e443: 8 total, 8 up, 8 in 2024-09-13T00:42:19.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:18 smithi088 bash[68960]: cluster 2024-09-13T00:42:18.402750+0000 mgr.y (mgr.25304) 158 : cluster [DBG] pgmap v256: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:42:19.102 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:42:18 smithi088 bash[61791]: ts=2024-09-13T00:42:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)\", cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", cluster_addr=\"172.21.15.77\", device_class=\"ssd\", hostname=\"smithi077\", instance=\"ceph_cluster\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.77\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:19.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:18 smithi077 bash[80681]: cluster 2024-09-13T00:42:17.889111+0000 mon.a (mon.0) 440 : cluster [DBG] osdmap e443: 8 total, 8 up, 8 in 2024-09-13T00:42:19.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:18 smithi077 bash[80681]: cluster 2024-09-13T00:42:18.402750+0000 mgr.y (mgr.25304) 158 : cluster [DBG] pgmap v256: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:42:19.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:18 smithi077 bash[83127]: cluster 2024-09-13T00:42:17.889111+0000 mon.a (mon.0) 440 : cluster [DBG] osdmap e443: 8 total, 8 up, 8 in 2024-09-13T00:42:19.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:18 smithi077 bash[83127]: cluster 2024-09-13T00:42:18.402750+0000 mgr.y (mgr.25304) 158 : cluster [DBG] pgmap v256: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:42:20.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:19 smithi077 bash[80681]: cluster 2024-09-13T00:42:18.909602+0000 mon.a (mon.0) 441 : cluster [DBG] osdmap e444: 8 total, 8 up, 8 in 2024-09-13T00:42:20.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:19 smithi077 bash[83127]: cluster 2024-09-13T00:42:18.909602+0000 mon.a (mon.0) 441 : cluster [DBG] osdmap e444: 8 total, 8 up, 8 in 2024-09-13T00:42:20.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:19 smithi088 bash[68960]: cluster 2024-09-13T00:42:18.909602+0000 mon.a (mon.0) 441 : cluster [DBG] osdmap e444: 8 total, 8 up, 8 in 2024-09-13T00:42:21.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:20 smithi077 bash[80681]: cluster 2024-09-13T00:42:19.909695+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e445: 8 total, 8 up, 8 in 2024-09-13T00:42:21.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:20 smithi077 bash[80681]: cluster 2024-09-13T00:42:20.403428+0000 mgr.y (mgr.25304) 159 : cluster [DBG] pgmap v259: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 7.2 KiB/s wr, 100 op/s 2024-09-13T00:42:21.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:20 smithi077 bash[83127]: cluster 2024-09-13T00:42:19.909695+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e445: 8 total, 8 up, 8 in 2024-09-13T00:42:21.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:20 smithi077 bash[83127]: cluster 2024-09-13T00:42:20.403428+0000 mgr.y (mgr.25304) 159 : cluster [DBG] pgmap v259: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 7.2 KiB/s wr, 100 op/s 2024-09-13T00:42:21.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:20 smithi088 bash[68960]: cluster 2024-09-13T00:42:19.909695+0000 mon.a (mon.0) 442 : cluster [DBG] osdmap e445: 8 total, 8 up, 8 in 2024-09-13T00:42:21.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:20 smithi088 bash[68960]: cluster 2024-09-13T00:42:20.403428+0000 mgr.y (mgr.25304) 159 : cluster [DBG] pgmap v259: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 7.2 KiB/s wr, 100 op/s 2024-09-13T00:42:22.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:21 smithi077 bash[80681]: cluster 2024-09-13T00:42:20.932306+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e446: 8 total, 8 up, 8 in 2024-09-13T00:42:22.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:21 smithi077 bash[83127]: cluster 2024-09-13T00:42:20.932306+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e446: 8 total, 8 up, 8 in 2024-09-13T00:42:22.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:21 smithi088 bash[68960]: cluster 2024-09-13T00:42:20.932306+0000 mon.a (mon.0) 443 : cluster [DBG] osdmap e446: 8 total, 8 up, 8 in 2024-09-13T00:42:22.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:42:22 smithi088 bash[61791]: ts=2024-09-13T00:42:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:23.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:22 smithi077 bash[80681]: cluster 2024-09-13T00:42:21.940591+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e447: 8 total, 8 up, 8 in 2024-09-13T00:42:23.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:22 smithi077 bash[80681]: cluster 2024-09-13T00:42:22.404138+0000 mgr.y (mgr.25304) 160 : cluster [DBG] pgmap v262: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 7.2 KiB/s wr, 100 op/s 2024-09-13T00:42:23.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:22 smithi077 bash[83127]: cluster 2024-09-13T00:42:21.940591+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e447: 8 total, 8 up, 8 in 2024-09-13T00:42:23.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:22 smithi077 bash[83127]: cluster 2024-09-13T00:42:22.404138+0000 mgr.y (mgr.25304) 160 : cluster [DBG] pgmap v262: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 7.2 KiB/s wr, 100 op/s 2024-09-13T00:42:23.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:22 smithi088 bash[68960]: cluster 2024-09-13T00:42:21.940591+0000 mon.a (mon.0) 444 : cluster [DBG] osdmap e447: 8 total, 8 up, 8 in 2024-09-13T00:42:23.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:22 smithi088 bash[68960]: cluster 2024-09-13T00:42:22.404138+0000 mgr.y (mgr.25304) 160 : cluster [DBG] pgmap v262: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 7.2 KiB/s wr, 100 op/s 2024-09-13T00:42:24.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:23 smithi077 bash[80681]: cluster 2024-09-13T00:42:22.946545+0000 mon.a (mon.0) 445 : cluster [DBG] osdmap e448: 8 total, 8 up, 8 in 2024-09-13T00:42:24.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:23 smithi077 bash[80681]: audit 2024-09-13T00:42:23.380585+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:24.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:23 smithi077 bash[80681]: audit 2024-09-13T00:42:23.387173+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:24.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:23 smithi077 bash[80681]: audit 2024-09-13T00:42:23.782638+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:24.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:23 smithi077 bash[80681]: audit 2024-09-13T00:42:23.783988+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:42:24.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:23 smithi077 bash[80681]: audit 2024-09-13T00:42:23.794392+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:24.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:23 smithi077 bash[83127]: cluster 2024-09-13T00:42:22.946545+0000 mon.a (mon.0) 445 : cluster [DBG] osdmap e448: 8 total, 8 up, 8 in 2024-09-13T00:42:24.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:23 smithi077 bash[83127]: audit 2024-09-13T00:42:23.380585+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:24.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:23 smithi077 bash[83127]: audit 2024-09-13T00:42:23.387173+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:24.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:23 smithi077 bash[83127]: audit 2024-09-13T00:42:23.782638+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:24.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:23 smithi077 bash[83127]: audit 2024-09-13T00:42:23.783988+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:42:24.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:23 smithi077 bash[83127]: audit 2024-09-13T00:42:23.794392+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:23 smithi088 bash[68960]: cluster 2024-09-13T00:42:22.946545+0000 mon.a (mon.0) 445 : cluster [DBG] osdmap e448: 8 total, 8 up, 8 in 2024-09-13T00:42:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:23 smithi088 bash[68960]: audit 2024-09-13T00:42:23.380585+0000 mon.a (mon.0) 446 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:23 smithi088 bash[68960]: audit 2024-09-13T00:42:23.387173+0000 mon.a (mon.0) 447 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:24.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:23 smithi088 bash[68960]: audit 2024-09-13T00:42:23.782638+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:24.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:23 smithi088 bash[68960]: audit 2024-09-13T00:42:23.783988+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:42:24.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:23 smithi088 bash[68960]: audit 2024-09-13T00:42:23.794392+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:25.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:24 smithi077 bash[80681]: cluster 2024-09-13T00:42:23.951502+0000 mon.a (mon.0) 451 : cluster [DBG] osdmap e449: 8 total, 8 up, 8 in 2024-09-13T00:42:25.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:24 smithi077 bash[80681]: cluster 2024-09-13T00:42:24.405384+0000 mgr.y (mgr.25304) 161 : cluster [DBG] pgmap v265: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 6.0 KiB/s wr, 86 op/s 2024-09-13T00:42:25.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:24 smithi077 bash[83127]: cluster 2024-09-13T00:42:23.951502+0000 mon.a (mon.0) 451 : cluster [DBG] osdmap e449: 8 total, 8 up, 8 in 2024-09-13T00:42:25.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:24 smithi077 bash[83127]: cluster 2024-09-13T00:42:24.405384+0000 mgr.y (mgr.25304) 161 : cluster [DBG] pgmap v265: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 6.0 KiB/s wr, 86 op/s 2024-09-13T00:42:25.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:24 smithi088 bash[68960]: cluster 2024-09-13T00:42:23.951502+0000 mon.a (mon.0) 451 : cluster [DBG] osdmap e449: 8 total, 8 up, 8 in 2024-09-13T00:42:25.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:24 smithi088 bash[68960]: cluster 2024-09-13T00:42:24.405384+0000 mgr.y (mgr.25304) 161 : cluster [DBG] pgmap v265: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 6.0 KiB/s wr, 86 op/s 2024-09-13T00:42:26.091 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.Snapshot (15563 ms) 2024-09-13T00:42:26.091 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.SnapshotRemoveOnDisable 2024-09-13T00:42:26.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:25 smithi077 bash[80681]: cluster 2024-09-13T00:42:24.980185+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e450: 8 total, 8 up, 8 in 2024-09-13T00:42:26.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:25 smithi077 bash[80681]: cluster 2024-09-13T00:42:25.884256+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e451: 8 total, 8 up, 8 in 2024-09-13T00:42:26.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:25 smithi077 bash[83127]: cluster 2024-09-13T00:42:24.980185+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e450: 8 total, 8 up, 8 in 2024-09-13T00:42:26.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:25 smithi077 bash[83127]: cluster 2024-09-13T00:42:25.884256+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e451: 8 total, 8 up, 8 in 2024-09-13T00:42:26.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:25 smithi088 bash[68960]: cluster 2024-09-13T00:42:24.980185+0000 mon.a (mon.0) 452 : cluster [DBG] osdmap e450: 8 total, 8 up, 8 in 2024-09-13T00:42:26.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:25 smithi088 bash[68960]: cluster 2024-09-13T00:42:25.884256+0000 mon.a (mon.0) 453 : cluster [DBG] osdmap e451: 8 total, 8 up, 8 in 2024-09-13T00:42:27.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:26 smithi077 bash[83127]: audit 2024-09-13T00:42:26.073178+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/bbedb173-a059-40cd-b64b-3715ab4abe95"}]: dispatch 2024-09-13T00:42:27.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:26 smithi077 bash[83127]: audit 2024-09-13T00:42:26.073976+0000 mon.a (mon.0) 454 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/bbedb173-a059-40cd-b64b-3715ab4abe95"}]: dispatch 2024-09-13T00:42:27.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:26 smithi077 bash[83127]: audit 2024-09-13T00:42:26.081383+0000 mon.a (mon.0) 455 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/bbedb173-a059-40cd-b64b-3715ab4abe95"}]': finished 2024-09-13T00:42:27.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:26 smithi077 bash[83127]: cluster 2024-09-13T00:42:26.406072+0000 mgr.y (mgr.25304) 162 : cluster [DBG] pgmap v268: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 6.0 KiB/s wr, 86 op/s 2024-09-13T00:42:27.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:26 smithi077 bash[80681]: audit 2024-09-13T00:42:26.073178+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/bbedb173-a059-40cd-b64b-3715ab4abe95"}]: dispatch 2024-09-13T00:42:27.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:26 smithi077 bash[80681]: audit 2024-09-13T00:42:26.073976+0000 mon.a (mon.0) 454 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/bbedb173-a059-40cd-b64b-3715ab4abe95"}]: dispatch 2024-09-13T00:42:27.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:26 smithi077 bash[80681]: audit 2024-09-13T00:42:26.081383+0000 mon.a (mon.0) 455 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/bbedb173-a059-40cd-b64b-3715ab4abe95"}]': finished 2024-09-13T00:42:27.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:26 smithi077 bash[80681]: cluster 2024-09-13T00:42:26.406072+0000 mgr.y (mgr.25304) 162 : cluster [DBG] pgmap v268: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 6.0 KiB/s wr, 86 op/s 2024-09-13T00:42:27.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:26 smithi088 bash[68960]: audit 2024-09-13T00:42:26.073178+0000 mon.c (mon.2) 11 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/bbedb173-a059-40cd-b64b-3715ab4abe95"}]: dispatch 2024-09-13T00:42:27.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:26 smithi088 bash[68960]: audit 2024-09-13T00:42:26.073976+0000 mon.a (mon.0) 454 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/bbedb173-a059-40cd-b64b-3715ab4abe95"}]: dispatch 2024-09-13T00:42:27.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:26 smithi088 bash[68960]: audit 2024-09-13T00:42:26.081383+0000 mon.a (mon.0) 455 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/bbedb173-a059-40cd-b64b-3715ab4abe95"}]': finished 2024-09-13T00:42:27.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:26 smithi088 bash[68960]: cluster 2024-09-13T00:42:26.406072+0000 mgr.y (mgr.25304) 162 : cluster [DBG] pgmap v268: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 62 KiB/s rd, 6.0 KiB/s wr, 86 op/s 2024-09-13T00:42:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:28 smithi088 bash[68960]: cluster 2024-09-13T00:42:27.095308+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e452: 8 total, 8 up, 8 in 2024-09-13T00:42:28.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:28 smithi077 bash[80681]: cluster 2024-09-13T00:42:27.095308+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e452: 8 total, 8 up, 8 in 2024-09-13T00:42:28.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:28 smithi077 bash[83127]: cluster 2024-09-13T00:42:27.095308+0000 mon.a (mon.0) 456 : cluster [DBG] osdmap e452: 8 total, 8 up, 8 in 2024-09-13T00:42:28.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:42:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:42:28] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.43.0" 2024-09-13T00:42:29.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:29 smithi077 bash[80681]: cluster 2024-09-13T00:42:28.113113+0000 mon.a (mon.0) 457 : cluster [DBG] osdmap e453: 8 total, 8 up, 8 in 2024-09-13T00:42:29.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:29 smithi077 bash[80681]: cluster 2024-09-13T00:42:28.406806+0000 mgr.y (mgr.25304) 163 : cluster [DBG] pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:42:29.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:29 smithi077 bash[80681]: audit 2024-09-13T00:42:28.413288+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:29.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:29 smithi077 bash[83127]: cluster 2024-09-13T00:42:28.113113+0000 mon.a (mon.0) 457 : cluster [DBG] osdmap e453: 8 total, 8 up, 8 in 2024-09-13T00:42:29.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:29 smithi077 bash[83127]: cluster 2024-09-13T00:42:28.406806+0000 mgr.y (mgr.25304) 163 : cluster [DBG] pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:42:29.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:29 smithi077 bash[83127]: audit 2024-09-13T00:42:28.413288+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:29.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:29 smithi088 bash[68960]: cluster 2024-09-13T00:42:28.113113+0000 mon.a (mon.0) 457 : cluster [DBG] osdmap e453: 8 total, 8 up, 8 in 2024-09-13T00:42:29.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:29 smithi088 bash[68960]: cluster 2024-09-13T00:42:28.406806+0000 mgr.y (mgr.25304) 163 : cluster [DBG] pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:42:29.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:29 smithi088 bash[68960]: audit 2024-09-13T00:42:28.413288+0000 mon.a (mon.0) 458 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:30.337 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.SnapshotRemoveOnDisable (4247 ms) 2024-09-13T00:42:30.338 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.SnapshotUnlinkPeer 2024-09-13T00:42:30.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:30 smithi077 bash[80681]: cluster 2024-09-13T00:42:29.123371+0000 mon.a (mon.0) 459 : cluster [DBG] osdmap e454: 8 total, 8 up, 8 in 2024-09-13T00:42:30.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:30 smithi077 bash[83127]: cluster 2024-09-13T00:42:29.123371+0000 mon.a (mon.0) 459 : cluster [DBG] osdmap e454: 8 total, 8 up, 8 in 2024-09-13T00:42:30.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:30 smithi088 bash[68960]: cluster 2024-09-13T00:42:29.123371+0000 mon.a (mon.0) 459 : cluster [DBG] osdmap e454: 8 total, 8 up, 8 in 2024-09-13T00:42:31.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:31 smithi077 bash[80681]: cluster 2024-09-13T00:42:30.125327+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e455: 8 total, 8 up, 8 in 2024-09-13T00:42:31.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:31 smithi077 bash[80681]: audit 2024-09-13T00:42:30.320915+0000 mon.c (mon.2) 12 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/a782e8dc-cef1-4960-8cd4-d8b452f09648"}]: dispatch 2024-09-13T00:42:31.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:31 smithi077 bash[80681]: audit 2024-09-13T00:42:30.321705+0000 mon.a (mon.0) 461 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/a782e8dc-cef1-4960-8cd4-d8b452f09648"}]: dispatch 2024-09-13T00:42:31.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:31 smithi077 bash[80681]: audit 2024-09-13T00:42:30.328210+0000 mon.a (mon.0) 462 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/a782e8dc-cef1-4960-8cd4-d8b452f09648"}]': finished 2024-09-13T00:42:31.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:31 smithi077 bash[80681]: cluster 2024-09-13T00:42:30.407532+0000 mgr.y (mgr.25304) 164 : cluster [DBG] pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 9.5 KiB/s wr, 110 op/s 2024-09-13T00:42:31.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:31 smithi077 bash[83127]: cluster 2024-09-13T00:42:30.125327+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e455: 8 total, 8 up, 8 in 2024-09-13T00:42:31.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:31 smithi077 bash[83127]: audit 2024-09-13T00:42:30.320915+0000 mon.c (mon.2) 12 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/a782e8dc-cef1-4960-8cd4-d8b452f09648"}]: dispatch 2024-09-13T00:42:31.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:31 smithi077 bash[83127]: audit 2024-09-13T00:42:30.321705+0000 mon.a (mon.0) 461 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/a782e8dc-cef1-4960-8cd4-d8b452f09648"}]: dispatch 2024-09-13T00:42:31.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:31 smithi077 bash[83127]: audit 2024-09-13T00:42:30.328210+0000 mon.a (mon.0) 462 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/a782e8dc-cef1-4960-8cd4-d8b452f09648"}]': finished 2024-09-13T00:42:31.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:31 smithi077 bash[83127]: cluster 2024-09-13T00:42:30.407532+0000 mgr.y (mgr.25304) 164 : cluster [DBG] pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 9.5 KiB/s wr, 110 op/s 2024-09-13T00:42:31.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:31 smithi088 bash[68960]: cluster 2024-09-13T00:42:30.125327+0000 mon.a (mon.0) 460 : cluster [DBG] osdmap e455: 8 total, 8 up, 8 in 2024-09-13T00:42:31.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:31 smithi088 bash[68960]: audit 2024-09-13T00:42:30.320915+0000 mon.c (mon.2) 12 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/a782e8dc-cef1-4960-8cd4-d8b452f09648"}]: dispatch 2024-09-13T00:42:31.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:31 smithi088 bash[68960]: audit 2024-09-13T00:42:30.321705+0000 mon.a (mon.0) 461 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/a782e8dc-cef1-4960-8cd4-d8b452f09648"}]: dispatch 2024-09-13T00:42:31.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:31 smithi088 bash[68960]: audit 2024-09-13T00:42:30.328210+0000 mon.a (mon.0) 462 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/a782e8dc-cef1-4960-8cd4-d8b452f09648"}]': finished 2024-09-13T00:42:31.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:31 smithi088 bash[68960]: cluster 2024-09-13T00:42:30.407532+0000 mgr.y (mgr.25304) 164 : cluster [DBG] pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 9.5 KiB/s wr, 110 op/s 2024-09-13T00:42:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:32 smithi088 bash[68960]: cluster 2024-09-13T00:42:31.340589+0000 mon.a (mon.0) 463 : cluster [DBG] osdmap e456: 8 total, 8 up, 8 in 2024-09-13T00:42:32.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:42:32 smithi088 bash[61791]: ts=2024-09-13T00:42:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:32.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:32 smithi077 bash[80681]: cluster 2024-09-13T00:42:31.340589+0000 mon.a (mon.0) 463 : cluster [DBG] osdmap e456: 8 total, 8 up, 8 in 2024-09-13T00:42:32.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:32 smithi077 bash[83127]: cluster 2024-09-13T00:42:31.340589+0000 mon.a (mon.0) 463 : cluster [DBG] osdmap e456: 8 total, 8 up, 8 in 2024-09-13T00:42:32.872 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph orch upgrade resume' 2024-09-13T00:42:33.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:33 smithi077 bash[80681]: cluster 2024-09-13T00:42:32.355905+0000 mon.a (mon.0) 464 : cluster [DBG] osdmap e457: 8 total, 8 up, 8 in 2024-09-13T00:42:33.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:33 smithi077 bash[80681]: cluster 2024-09-13T00:42:32.408226+0000 mgr.y (mgr.25304) 165 : cluster [DBG] pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 9.5 KiB/s wr, 110 op/s 2024-09-13T00:42:33.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:33 smithi077 bash[83127]: cluster 2024-09-13T00:42:32.355905+0000 mon.a (mon.0) 464 : cluster [DBG] osdmap e457: 8 total, 8 up, 8 in 2024-09-13T00:42:33.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:33 smithi077 bash[83127]: cluster 2024-09-13T00:42:32.408226+0000 mgr.y (mgr.25304) 165 : cluster [DBG] pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 9.5 KiB/s wr, 110 op/s 2024-09-13T00:42:33.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:33 smithi088 bash[68960]: cluster 2024-09-13T00:42:32.355905+0000 mon.a (mon.0) 464 : cluster [DBG] osdmap e457: 8 total, 8 up, 8 in 2024-09-13T00:42:33.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:33 smithi088 bash[68960]: cluster 2024-09-13T00:42:32.408226+0000 mgr.y (mgr.25304) 165 : cluster [DBG] pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 9.5 KiB/s wr, 110 op/s 2024-09-13T00:42:34.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:34 smithi077 bash[80681]: cluster 2024-09-13T00:42:33.362417+0000 mon.a (mon.0) 465 : cluster [DBG] osdmap e458: 8 total, 8 up, 8 in 2024-09-13T00:42:34.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:34 smithi077 bash[83127]: cluster 2024-09-13T00:42:33.362417+0000 mon.a (mon.0) 465 : cluster [DBG] osdmap e458: 8 total, 8 up, 8 in 2024-09-13T00:42:34.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:34 smithi088 bash[68960]: cluster 2024-09-13T00:42:33.362417+0000 mon.a (mon.0) 465 : cluster [DBG] osdmap e458: 8 total, 8 up, 8 in 2024-09-13T00:42:35.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:35 smithi077 bash[80681]: cluster 2024-09-13T00:42:34.363673+0000 mon.a (mon.0) 466 : cluster [DBG] osdmap e459: 8 total, 8 up, 8 in 2024-09-13T00:42:35.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:35 smithi077 bash[80681]: cluster 2024-09-13T00:42:34.409252+0000 mgr.y (mgr.25304) 166 : cluster [DBG] pgmap v280: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 107 KiB/s rd, 9.7 KiB/s wr, 146 op/s 2024-09-13T00:42:35.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:35 smithi077 bash[83127]: cluster 2024-09-13T00:42:34.363673+0000 mon.a (mon.0) 466 : cluster [DBG] osdmap e459: 8 total, 8 up, 8 in 2024-09-13T00:42:35.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:35 smithi077 bash[83127]: cluster 2024-09-13T00:42:34.409252+0000 mgr.y (mgr.25304) 166 : cluster [DBG] pgmap v280: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 107 KiB/s rd, 9.7 KiB/s wr, 146 op/s 2024-09-13T00:42:35.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:35 smithi088 bash[68960]: cluster 2024-09-13T00:42:34.363673+0000 mon.a (mon.0) 466 : cluster [DBG] osdmap e459: 8 total, 8 up, 8 in 2024-09-13T00:42:35.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:35 smithi088 bash[68960]: cluster 2024-09-13T00:42:34.409252+0000 mgr.y (mgr.25304) 166 : cluster [DBG] pgmap v280: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 107 KiB/s rd, 9.7 KiB/s wr, 146 op/s 2024-09-13T00:42:36.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:36 smithi077 bash[83127]: cluster 2024-09-13T00:42:35.391340+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e460: 8 total, 8 up, 8 in 2024-09-13T00:42:36.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:36 smithi077 bash[83127]: audit 2024-09-13T00:42:35.427169+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e04899eb-b65e-42cf-bdc2-1a9e82a4ec27"}]: dispatch 2024-09-13T00:42:36.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:36 smithi077 bash[83127]: audit 2024-09-13T00:42:35.427871+0000 mon.a (mon.0) 468 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e04899eb-b65e-42cf-bdc2-1a9e82a4ec27"}]: dispatch 2024-09-13T00:42:36.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:36 smithi077 bash[83127]: audit 2024-09-13T00:42:35.434450+0000 mon.a (mon.0) 469 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e04899eb-b65e-42cf-bdc2-1a9e82a4ec27"}]': finished 2024-09-13T00:42:36.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:36 smithi077 bash[83127]: cluster 2024-09-13T00:42:35.887204+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e461: 8 total, 8 up, 8 in 2024-09-13T00:42:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:36 smithi077 bash[80681]: cluster 2024-09-13T00:42:35.391340+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e460: 8 total, 8 up, 8 in 2024-09-13T00:42:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:36 smithi077 bash[80681]: audit 2024-09-13T00:42:35.427169+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e04899eb-b65e-42cf-bdc2-1a9e82a4ec27"}]: dispatch 2024-09-13T00:42:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:36 smithi077 bash[80681]: audit 2024-09-13T00:42:35.427871+0000 mon.a (mon.0) 468 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e04899eb-b65e-42cf-bdc2-1a9e82a4ec27"}]: dispatch 2024-09-13T00:42:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:36 smithi077 bash[80681]: audit 2024-09-13T00:42:35.434450+0000 mon.a (mon.0) 469 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e04899eb-b65e-42cf-bdc2-1a9e82a4ec27"}]': finished 2024-09-13T00:42:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:36 smithi077 bash[80681]: cluster 2024-09-13T00:42:35.887204+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e461: 8 total, 8 up, 8 in 2024-09-13T00:42:36.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:36 smithi088 bash[68960]: cluster 2024-09-13T00:42:35.391340+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e460: 8 total, 8 up, 8 in 2024-09-13T00:42:36.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:36 smithi088 bash[68960]: audit 2024-09-13T00:42:35.427169+0000 mon.c (mon.2) 13 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e04899eb-b65e-42cf-bdc2-1a9e82a4ec27"}]: dispatch 2024-09-13T00:42:36.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:36 smithi088 bash[68960]: audit 2024-09-13T00:42:35.427871+0000 mon.a (mon.0) 468 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e04899eb-b65e-42cf-bdc2-1a9e82a4ec27"}]: dispatch 2024-09-13T00:42:36.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:36 smithi088 bash[68960]: audit 2024-09-13T00:42:35.434450+0000 mon.a (mon.0) 469 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/e04899eb-b65e-42cf-bdc2-1a9e82a4ec27"}]': finished 2024-09-13T00:42:36.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:36 smithi088 bash[68960]: cluster 2024-09-13T00:42:35.887204+0000 mon.a (mon.0) 470 : cluster [DBG] osdmap e461: 8 total, 8 up, 8 in 2024-09-13T00:42:37.661 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:42:37.757 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:37 smithi077 bash[80681]: cluster 2024-09-13T00:42:36.409976+0000 mgr.y (mgr.25304) 167 : cluster [DBG] pgmap v283: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 107 KiB/s rd, 9.7 KiB/s wr, 146 op/s 2024-09-13T00:42:37.757 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:37 smithi077 bash[80681]: cluster 2024-09-13T00:42:36.892514+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e462: 8 total, 8 up, 8 in 2024-09-13T00:42:37.757 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:37 smithi077 bash[83127]: cluster 2024-09-13T00:42:36.409976+0000 mgr.y (mgr.25304) 167 : cluster [DBG] pgmap v283: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 107 KiB/s rd, 9.7 KiB/s wr, 146 op/s 2024-09-13T00:42:37.757 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:37 smithi077 bash[83127]: cluster 2024-09-13T00:42:36.892514+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e462: 8 total, 8 up, 8 in 2024-09-13T00:42:37.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:37 smithi088 bash[68960]: cluster 2024-09-13T00:42:36.409976+0000 mgr.y (mgr.25304) 167 : cluster [DBG] pgmap v283: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 107 KiB/s rd, 9.7 KiB/s wr, 146 op/s 2024-09-13T00:42:37.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:37 smithi088 bash[68960]: cluster 2024-09-13T00:42:36.892514+0000 mon.a (mon.0) 471 : cluster [DBG] osdmap e462: 8 total, 8 up, 8 in 2024-09-13T00:42:38.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:42:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:42:38] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.43.0" 2024-09-13T00:42:39.179 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:38 smithi077 bash[83127]: cluster 2024-09-13T00:42:37.895142+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2024-09-13T00:42:39.180 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:38 smithi077 bash[83127]: cluster 2024-09-13T00:42:38.410665+0000 mgr.y (mgr.25304) 168 : cluster [DBG] pgmap v286: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:42:39.180 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:38 smithi077 bash[80681]: cluster 2024-09-13T00:42:37.895142+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2024-09-13T00:42:39.180 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:38 smithi077 bash[80681]: cluster 2024-09-13T00:42:38.410665+0000 mgr.y (mgr.25304) 168 : cluster [DBG] pgmap v286: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:42:39.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:38 smithi088 bash[68960]: cluster 2024-09-13T00:42:37.895142+0000 mon.a (mon.0) 472 : cluster [DBG] osdmap e463: 8 total, 8 up, 8 in 2024-09-13T00:42:39.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:38 smithi088 bash[68960]: cluster 2024-09-13T00:42:38.410665+0000 mgr.y (mgr.25304) 168 : cluster [DBG] pgmap v286: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:42:40.079 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.SnapshotUnlinkPeer (9741 ms) 2024-09-13T00:42:40.079 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.SnapshotImageState 2024-09-13T00:42:40.204 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:39 smithi077 bash[80681]: cluster 2024-09-13T00:42:38.905806+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2024-09-13T00:42:40.205 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:39 smithi077 bash[83127]: cluster 2024-09-13T00:42:38.905806+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2024-09-13T00:42:40.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:39 smithi088 bash[68960]: cluster 2024-09-13T00:42:38.905806+0000 mon.a (mon.0) 473 : cluster [DBG] osdmap e464: 8 total, 8 up, 8 in 2024-09-13T00:42:41.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:40 smithi077 bash[80681]: cluster 2024-09-13T00:42:39.907752+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2024-09-13T00:42:41.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:40 smithi077 bash[80681]: audit 2024-09-13T00:42:40.042521+0000 mon.c (mon.2) 14 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/27daf57a-fc7f-47a4-ac8e-46668c03ad06"}]: dispatch 2024-09-13T00:42:41.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:40 smithi077 bash[80681]: audit 2024-09-13T00:42:40.043208+0000 mon.a (mon.0) 475 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/27daf57a-fc7f-47a4-ac8e-46668c03ad06"}]: dispatch 2024-09-13T00:42:41.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:40 smithi077 bash[80681]: audit 2024-09-13T00:42:40.051480+0000 mon.a (mon.0) 476 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/27daf57a-fc7f-47a4-ac8e-46668c03ad06"}]': finished 2024-09-13T00:42:41.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:40 smithi077 bash[80681]: audit 2024-09-13T00:42:40.061490+0000 mon.c (mon.2) 15 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d8cfcaa5-366f-444e-b9fc-6b105afa511a"}]: dispatch 2024-09-13T00:42:41.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:40 smithi077 bash[80681]: audit 2024-09-13T00:42:40.063817+0000 mon.a (mon.0) 477 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d8cfcaa5-366f-444e-b9fc-6b105afa511a"}]: dispatch 2024-09-13T00:42:41.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:40 smithi077 bash[80681]: audit 2024-09-13T00:42:40.074251+0000 mon.a (mon.0) 478 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d8cfcaa5-366f-444e-b9fc-6b105afa511a"}]': finished 2024-09-13T00:42:41.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:40 smithi077 bash[80681]: cluster 2024-09-13T00:42:40.411367+0000 mgr.y (mgr.25304) 169 : cluster [DBG] pgmap v289: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 11 KiB/s wr, 227 op/s 2024-09-13T00:42:41.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:40 smithi077 bash[83127]: cluster 2024-09-13T00:42:39.907752+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2024-09-13T00:42:41.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:40 smithi077 bash[83127]: audit 2024-09-13T00:42:40.042521+0000 mon.c (mon.2) 14 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/27daf57a-fc7f-47a4-ac8e-46668c03ad06"}]: dispatch 2024-09-13T00:42:41.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:40 smithi077 bash[83127]: audit 2024-09-13T00:42:40.043208+0000 mon.a (mon.0) 475 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/27daf57a-fc7f-47a4-ac8e-46668c03ad06"}]: dispatch 2024-09-13T00:42:41.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:40 smithi077 bash[83127]: audit 2024-09-13T00:42:40.051480+0000 mon.a (mon.0) 476 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/27daf57a-fc7f-47a4-ac8e-46668c03ad06"}]': finished 2024-09-13T00:42:41.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:40 smithi077 bash[83127]: audit 2024-09-13T00:42:40.061490+0000 mon.c (mon.2) 15 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d8cfcaa5-366f-444e-b9fc-6b105afa511a"}]: dispatch 2024-09-13T00:42:41.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:40 smithi077 bash[83127]: audit 2024-09-13T00:42:40.063817+0000 mon.a (mon.0) 477 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d8cfcaa5-366f-444e-b9fc-6b105afa511a"}]: dispatch 2024-09-13T00:42:41.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:40 smithi077 bash[83127]: audit 2024-09-13T00:42:40.074251+0000 mon.a (mon.0) 478 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d8cfcaa5-366f-444e-b9fc-6b105afa511a"}]': finished 2024-09-13T00:42:41.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:40 smithi077 bash[83127]: cluster 2024-09-13T00:42:40.411367+0000 mgr.y (mgr.25304) 169 : cluster [DBG] pgmap v289: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 11 KiB/s wr, 227 op/s 2024-09-13T00:42:41.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:40 smithi088 bash[68960]: cluster 2024-09-13T00:42:39.907752+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e465: 8 total, 8 up, 8 in 2024-09-13T00:42:41.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:40 smithi088 bash[68960]: audit 2024-09-13T00:42:40.042521+0000 mon.c (mon.2) 14 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/27daf57a-fc7f-47a4-ac8e-46668c03ad06"}]: dispatch 2024-09-13T00:42:41.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:40 smithi088 bash[68960]: audit 2024-09-13T00:42:40.043208+0000 mon.a (mon.0) 475 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/27daf57a-fc7f-47a4-ac8e-46668c03ad06"}]: dispatch 2024-09-13T00:42:41.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:40 smithi088 bash[68960]: audit 2024-09-13T00:42:40.051480+0000 mon.a (mon.0) 476 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/27daf57a-fc7f-47a4-ac8e-46668c03ad06"}]': finished 2024-09-13T00:42:41.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:40 smithi088 bash[68960]: audit 2024-09-13T00:42:40.061490+0000 mon.c (mon.2) 15 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d8cfcaa5-366f-444e-b9fc-6b105afa511a"}]: dispatch 2024-09-13T00:42:41.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:40 smithi088 bash[68960]: audit 2024-09-13T00:42:40.063817+0000 mon.a (mon.0) 477 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d8cfcaa5-366f-444e-b9fc-6b105afa511a"}]: dispatch 2024-09-13T00:42:41.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:40 smithi088 bash[68960]: audit 2024-09-13T00:42:40.074251+0000 mon.a (mon.0) 478 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/d8cfcaa5-366f-444e-b9fc-6b105afa511a"}]': finished 2024-09-13T00:42:41.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:40 smithi088 bash[68960]: cluster 2024-09-13T00:42:40.411367+0000 mgr.y (mgr.25304) 169 : cluster [DBG] pgmap v289: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 11 KiB/s wr, 227 op/s 2024-09-13T00:42:41.724 INFO:teuthology.orchestra.run.smithi077.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:42:42.351 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:42:42 smithi088 bash[61791]: ts=2024-09-13T00:42:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:42.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:42 smithi088 bash[68960]: cluster 2024-09-13T00:42:41.088671+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2024-09-13T00:42:42.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:42 smithi088 bash[68960]: audit 2024-09-13T00:42:41.727164+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:42.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:42 smithi088 bash[68960]: audit 2024-09-13T00:42:41.728165+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:42:42.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:42 smithi077 bash[80681]: cluster 2024-09-13T00:42:41.088671+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2024-09-13T00:42:42.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:42 smithi077 bash[80681]: audit 2024-09-13T00:42:41.727164+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:42.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:42 smithi077 bash[80681]: audit 2024-09-13T00:42:41.728165+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:42:42.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:42 smithi077 bash[83127]: cluster 2024-09-13T00:42:41.088671+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e466: 8 total, 8 up, 8 in 2024-09-13T00:42:42.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:42 smithi077 bash[83127]: audit 2024-09-13T00:42:41.727164+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:42.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:42 smithi077 bash[83127]: audit 2024-09-13T00:42:41.728165+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:42:42.629 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'echo wait for all mons to upgrade' 2024-09-13T00:42:43.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:43 smithi077 bash[80681]: audit 2024-09-13T00:42:41.720200+0000 mgr.y (mgr.25304) 170 : audit [DBG] from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:42:43.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:43 smithi077 bash[80681]: cephadm 2024-09-13T00:42:41.720827+0000 mgr.y (mgr.25304) 171 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:42:43.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:43 smithi077 bash[80681]: cluster 2024-09-13T00:42:42.100683+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2024-09-13T00:42:43.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:43 smithi077 bash[80681]: cluster 2024-09-13T00:42:42.412043+0000 mgr.y (mgr.25304) 172 : cluster [DBG] pgmap v292: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 11 KiB/s wr, 227 op/s 2024-09-13T00:42:43.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:43 smithi077 bash[83127]: audit 2024-09-13T00:42:41.720200+0000 mgr.y (mgr.25304) 170 : audit [DBG] from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:42:43.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:43 smithi077 bash[83127]: cephadm 2024-09-13T00:42:41.720827+0000 mgr.y (mgr.25304) 171 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:42:43.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:43 smithi077 bash[83127]: cluster 2024-09-13T00:42:42.100683+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2024-09-13T00:42:43.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:43 smithi077 bash[83127]: cluster 2024-09-13T00:42:42.412043+0000 mgr.y (mgr.25304) 172 : cluster [DBG] pgmap v292: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 11 KiB/s wr, 227 op/s 2024-09-13T00:42:43.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:43 smithi088 bash[68960]: audit 2024-09-13T00:42:41.720200+0000 mgr.y (mgr.25304) 170 : audit [DBG] from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:42:43.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:43 smithi088 bash[68960]: cephadm 2024-09-13T00:42:41.720827+0000 mgr.y (mgr.25304) 171 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:42:43.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:43 smithi088 bash[68960]: cluster 2024-09-13T00:42:42.100683+0000 mon.a (mon.0) 482 : cluster [DBG] osdmap e467: 8 total, 8 up, 8 in 2024-09-13T00:42:43.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:43 smithi088 bash[68960]: cluster 2024-09-13T00:42:42.412043+0000 mgr.y (mgr.25304) 172 : cluster [DBG] pgmap v292: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 11 KiB/s wr, 227 op/s 2024-09-13T00:42:44.231 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.SnapshotImageState (4152 ms) 2024-09-13T00:42:44.231 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.SnapshotPromoteDemote 2024-09-13T00:42:44.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:44 smithi077 bash[80681]: cluster 2024-09-13T00:42:43.110937+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2024-09-13T00:42:44.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:44 smithi077 bash[80681]: audit 2024-09-13T00:42:43.413521+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:44.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:44 smithi077 bash[83127]: cluster 2024-09-13T00:42:43.110937+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2024-09-13T00:42:44.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:44 smithi077 bash[83127]: audit 2024-09-13T00:42:43.413521+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:44.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:44 smithi088 bash[68960]: cluster 2024-09-13T00:42:43.110937+0000 mon.a (mon.0) 483 : cluster [DBG] osdmap e468: 8 total, 8 up, 8 in 2024-09-13T00:42:44.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:44 smithi088 bash[68960]: audit 2024-09-13T00:42:43.413521+0000 mon.a (mon.0) 484 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:45.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:45 smithi077 bash[80681]: cluster 2024-09-13T00:42:44.118922+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2024-09-13T00:42:45.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:45 smithi077 bash[80681]: cluster 2024-09-13T00:42:44.413343+0000 mgr.y (mgr.25304) 173 : cluster [DBG] pgmap v295: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T00:42:45.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:45 smithi077 bash[83127]: cluster 2024-09-13T00:42:44.118922+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2024-09-13T00:42:45.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:45 smithi077 bash[83127]: cluster 2024-09-13T00:42:44.413343+0000 mgr.y (mgr.25304) 173 : cluster [DBG] pgmap v295: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T00:42:45.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:45 smithi088 bash[68960]: cluster 2024-09-13T00:42:44.118922+0000 mon.a (mon.0) 485 : cluster [DBG] osdmap e469: 8 total, 8 up, 8 in 2024-09-13T00:42:45.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:45 smithi088 bash[68960]: cluster 2024-09-13T00:42:44.413343+0000 mgr.y (mgr.25304) 173 : cluster [DBG] pgmap v295: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T00:42:46.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:46 smithi077 bash[80681]: cluster 2024-09-13T00:42:45.130756+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2024-09-13T00:42:46.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:46 smithi077 bash[80681]: cluster 2024-09-13T00:42:45.897862+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e471: 8 total, 8 up, 8 in 2024-09-13T00:42:46.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:46 smithi077 bash[83127]: cluster 2024-09-13T00:42:45.130756+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2024-09-13T00:42:46.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:46 smithi077 bash[83127]: cluster 2024-09-13T00:42:45.897862+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e471: 8 total, 8 up, 8 in 2024-09-13T00:42:46.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:46 smithi088 bash[68960]: cluster 2024-09-13T00:42:45.130756+0000 mon.a (mon.0) 486 : cluster [DBG] osdmap e470: 8 total, 8 up, 8 in 2024-09-13T00:42:46.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:46 smithi088 bash[68960]: cluster 2024-09-13T00:42:45.897862+0000 mon.a (mon.0) 487 : cluster [DBG] osdmap e471: 8 total, 8 up, 8 in 2024-09-13T00:42:47.430 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:42:47.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:47 smithi077 bash[83127]: cluster 2024-09-13T00:42:46.414156+0000 mgr.y (mgr.25304) 174 : cluster [DBG] pgmap v298: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T00:42:47.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:47 smithi077 bash[83127]: cluster 2024-09-13T00:42:46.911507+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e472: 8 total, 8 up, 8 in 2024-09-13T00:42:47.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:47 smithi077 bash[80681]: cluster 2024-09-13T00:42:46.414156+0000 mgr.y (mgr.25304) 174 : cluster [DBG] pgmap v298: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T00:42:47.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:47 smithi077 bash[80681]: cluster 2024-09-13T00:42:46.911507+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e472: 8 total, 8 up, 8 in 2024-09-13T00:42:47.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:47 smithi088 bash[68960]: cluster 2024-09-13T00:42:46.414156+0000 mgr.y (mgr.25304) 174 : cluster [DBG] pgmap v298: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-13T00:42:47.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:47 smithi088 bash[68960]: cluster 2024-09-13T00:42:46.911507+0000 mon.a (mon.0) 488 : cluster [DBG] osdmap e472: 8 total, 8 up, 8 in 2024-09-13T00:42:48.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:42:48] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.43.0" 2024-09-13T00:42:48.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:47.509498+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:47.517737+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: cluster 2024-09-13T00:42:47.914439+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e473: 8 total, 8 up, 8 in 2024-09-13T00:42:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:47.975959+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:47.977023+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:42:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:47.984100+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:48.069065+0000 mon.a (mon.0) 495 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:42:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:48.072685+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:42:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:48.075136+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:42:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: cephadm 2024-09-13T00:42:48.076326+0000 mgr.y (mgr.25304) 175 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-09-13T00:42:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:48.084898+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:48.086217+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T00:42:48.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:48.093976+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T00:42:48.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:48.095192+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T00:42:48.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:48.102739+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T00:42:48.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:48.104014+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T00:42:48.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:48.109120+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T00:42:48.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:48.111219+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:42:48.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: cephadm 2024-09-13T00:42:48.112837+0000 mgr.y (mgr.25304) 176 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-13T00:42:48.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:47.509498+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:48.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:47.517737+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:48.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: cluster 2024-09-13T00:42:47.914439+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e473: 8 total, 8 up, 8 in 2024-09-13T00:42:48.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:47.975959+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:48.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:47.977023+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:42:48.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:47.984100+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:48.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:48.069065+0000 mon.a (mon.0) 495 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:42:48.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:48.072685+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:42:48.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:48.075136+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:42:48.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: cephadm 2024-09-13T00:42:48.076326+0000 mgr.y (mgr.25304) 175 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-09-13T00:42:48.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:48.084898+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:48.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:48.086217+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T00:42:48.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:48.093976+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T00:42:48.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:48.095192+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T00:42:48.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:48.102739+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T00:42:48.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:48.104014+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T00:42:48.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:48.109120+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T00:42:48.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:48.111219+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:42:48.782 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: cephadm 2024-09-13T00:42:48.112837+0000 mgr.y (mgr.25304) 176 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-13T00:42:48.782 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:48.117386+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:48.782 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:48.119444+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T00:42:48.782 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: audit 2024-09-13T00:42:48.119863+0000 mgr.y (mgr.25304) 177 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T00:42:48.782 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: cephadm 2024-09-13T00:42:48.121811+0000 mgr.y (mgr.25304) 178 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-09-13T00:42:48.782 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[80681]: cluster 2024-09-13T00:42:48.414606+0000 mgr.y (mgr.25304) 179 : cluster [DBG] pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:42:48.783 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:48.117386+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:48.783 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:48.119444+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T00:42:48.783 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: audit 2024-09-13T00:42:48.119863+0000 mgr.y (mgr.25304) 177 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T00:42:48.783 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: cephadm 2024-09-13T00:42:48.121811+0000 mgr.y (mgr.25304) 178 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-09-13T00:42:48.783 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:48 smithi077 bash[83127]: cluster 2024-09-13T00:42:48.414606+0000 mgr.y (mgr.25304) 179 : cluster [DBG] pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:42:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:47.509498+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:47.517737+0000 mon.a (mon.0) 490 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: cluster 2024-09-13T00:42:47.914439+0000 mon.a (mon.0) 491 : cluster [DBG] osdmap e473: 8 total, 8 up, 8 in 2024-09-13T00:42:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:47.975959+0000 mon.a (mon.0) 492 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:47.977023+0000 mon.a (mon.0) 493 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:42:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:47.984100+0000 mon.a (mon.0) 494 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:48.069065+0000 mon.a (mon.0) 495 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:42:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:48.072685+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:42:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:48.075136+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:42:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: cephadm 2024-09-13T00:42:48.076326+0000 mgr.y (mgr.25304) 175 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-09-13T00:42:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:48.084898+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:48.086217+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-13T00:42:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:48.093976+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-13T00:42:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:48.095192+0000 mon.a (mon.0) 501 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-09-13T00:42:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:48.102739+0000 mon.a (mon.0) 502 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-13T00:42:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:48.104014+0000 mon.a (mon.0) 503 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-13T00:42:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:48.109120+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-13T00:42:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:48.111219+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:42:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: cephadm 2024-09-13T00:42:48.112837+0000 mgr.y (mgr.25304) 176 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-09-13T00:42:48.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:48.117386+0000 mon.a (mon.0) 506 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:48.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:48.119444+0000 mon.a (mon.0) 507 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T00:42:48.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: audit 2024-09-13T00:42:48.119863+0000 mgr.y (mgr.25304) 177 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-13T00:42:48.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: cephadm 2024-09-13T00:42:48.121811+0000 mgr.y (mgr.25304) 178 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-09-13T00:42:48.854 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:48 smithi088 bash[68960]: cluster 2024-09-13T00:42:48.414606+0000 mgr.y (mgr.25304) 179 : cluster [DBG] pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:42:50.116 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:49 smithi077 bash[80681]: cluster 2024-09-13T00:42:48.922578+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e474: 8 total, 8 up, 8 in 2024-09-13T00:42:50.116 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:49 smithi077 bash[83127]: cluster 2024-09-13T00:42:48.922578+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e474: 8 total, 8 up, 8 in 2024-09-13T00:42:50.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:49 smithi088 bash[68960]: cluster 2024-09-13T00:42:48.922578+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e474: 8 total, 8 up, 8 in 2024-09-13T00:42:51.478 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:51 smithi077 bash[80681]: cluster 2024-09-13T00:42:50.134107+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e475: 8 total, 8 up, 8 in 2024-09-13T00:42:51.478 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:51 smithi077 bash[80681]: cluster 2024-09-13T00:42:50.415170+0000 mgr.y (mgr.25304) 180 : cluster [DBG] pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T00:42:51.479 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:51 smithi077 bash[83127]: cluster 2024-09-13T00:42:50.134107+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e475: 8 total, 8 up, 8 in 2024-09-13T00:42:51.479 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:51 smithi077 bash[83127]: cluster 2024-09-13T00:42:50.415170+0000 mgr.y (mgr.25304) 180 : cluster [DBG] pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T00:42:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:51 smithi088 bash[68960]: cluster 2024-09-13T00:42:50.134107+0000 mon.a (mon.0) 509 : cluster [DBG] osdmap e475: 8 total, 8 up, 8 in 2024-09-13T00:42:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:51 smithi088 bash[68960]: cluster 2024-09-13T00:42:50.415170+0000 mgr.y (mgr.25304) 180 : cluster [DBG] pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T00:42:52.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:52 smithi077 bash[80681]: cluster 2024-09-13T00:42:51.137569+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e476: 8 total, 8 up, 8 in 2024-09-13T00:42:52.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:52 smithi077 bash[83127]: cluster 2024-09-13T00:42:51.137569+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e476: 8 total, 8 up, 8 in 2024-09-13T00:42:52.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:52 smithi088 bash[68960]: cluster 2024-09-13T00:42:51.137569+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e476: 8 total, 8 up, 8 in 2024-09-13T00:42:52.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:42:52 smithi088 bash[61791]: ts=2024-09-13T00:42:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:42:53.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:53 smithi077 bash[80681]: cluster 2024-09-13T00:42:52.150504+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e477: 8 total, 8 up, 8 in 2024-09-13T00:42:53.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:53 smithi077 bash[80681]: cluster 2024-09-13T00:42:52.415843+0000 mgr.y (mgr.25304) 181 : cluster [DBG] pgmap v307: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T00:42:53.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:53 smithi077 bash[80681]: audit 2024-09-13T00:42:52.749561+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:53.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:53 smithi077 bash[80681]: audit 2024-09-13T00:42:52.751110+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T00:42:53.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:53 smithi077 bash[80681]: audit 2024-09-13T00:42:52.752412+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:53.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:53 smithi077 bash[83127]: cluster 2024-09-13T00:42:52.150504+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e477: 8 total, 8 up, 8 in 2024-09-13T00:42:53.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:53 smithi077 bash[83127]: cluster 2024-09-13T00:42:52.415843+0000 mgr.y (mgr.25304) 181 : cluster [DBG] pgmap v307: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T00:42:53.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:53 smithi077 bash[83127]: audit 2024-09-13T00:42:52.749561+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:53.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:53 smithi077 bash[83127]: audit 2024-09-13T00:42:52.751110+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T00:42:53.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:53 smithi077 bash[83127]: audit 2024-09-13T00:42:52.752412+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:53.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:53 smithi088 bash[68960]: cluster 2024-09-13T00:42:52.150504+0000 mon.a (mon.0) 511 : cluster [DBG] osdmap e477: 8 total, 8 up, 8 in 2024-09-13T00:42:53.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:53 smithi088 bash[68960]: cluster 2024-09-13T00:42:52.415843+0000 mgr.y (mgr.25304) 181 : cluster [DBG] pgmap v307: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-13T00:42:53.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:53 smithi088 bash[68960]: audit 2024-09-13T00:42:52.749561+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:42:53.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:53 smithi088 bash[68960]: audit 2024-09-13T00:42:52.751110+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-13T00:42:53.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:53 smithi088 bash[68960]: audit 2024-09-13T00:42:52.752412+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:42:54.228 INFO:teuthology.orchestra.run.smithi077.stdout:wait for all mons to upgrade 2024-09-13T00:42:54.254 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.SnapshotPromoteDemote (10024 ms) 2024-09-13T00:42:54.255 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestMirroring.AioSnapshotCreate 2024-09-13T00:42:54.432 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:54 smithi077 bash[80681]: cephadm 2024-09-13T00:42:52.740266+0000 mgr.y (mgr.25304) 182 : cephadm [INF] Upgrade: Updating osd.0 2024-09-13T00:42:54.432 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:54 smithi077 bash[80681]: cephadm 2024-09-13T00:42:52.755453+0000 mgr.y (mgr.25304) 183 : cephadm [INF] Deploying daemon osd.0 on smithi077 2024-09-13T00:42:54.432 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:54 smithi077 bash[80681]: cluster 2024-09-13T00:42:53.168964+0000 mon.a (mon.0) 515 : cluster [DBG] osdmap e478: 8 total, 8 up, 8 in 2024-09-13T00:42:54.432 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:54 smithi077 bash[83127]: cephadm 2024-09-13T00:42:52.740266+0000 mgr.y (mgr.25304) 182 : cephadm [INF] Upgrade: Updating osd.0 2024-09-13T00:42:54.432 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:54 smithi077 bash[83127]: cephadm 2024-09-13T00:42:52.755453+0000 mgr.y (mgr.25304) 183 : cephadm [INF] Deploying daemon osd.0 on smithi077 2024-09-13T00:42:54.433 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:54 smithi077 bash[83127]: cluster 2024-09-13T00:42:53.168964+0000 mon.a (mon.0) 515 : cluster [DBG] osdmap e478: 8 total, 8 up, 8 in 2024-09-13T00:42:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:54 smithi088 bash[68960]: cephadm 2024-09-13T00:42:52.740266+0000 mgr.y (mgr.25304) 182 : cephadm [INF] Upgrade: Updating osd.0 2024-09-13T00:42:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:54 smithi088 bash[68960]: cephadm 2024-09-13T00:42:52.755453+0000 mgr.y (mgr.25304) 183 : cephadm [INF] Deploying daemon osd.0 on smithi077 2024-09-13T00:42:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:54 smithi088 bash[68960]: cluster 2024-09-13T00:42:53.168964+0000 mon.a (mon.0) 515 : cluster [DBG] osdmap e478: 8 total, 8 up, 8 in 2024-09-13T00:42:55.517 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:55 smithi077 bash[80681]: cluster 2024-09-13T00:42:54.172534+0000 mon.a (mon.0) 516 : cluster [DBG] osdmap e479: 8 total, 8 up, 8 in 2024-09-13T00:42:55.517 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:55 smithi077 bash[80681]: audit 2024-09-13T00:42:54.246716+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/88d10567-eb59-49b6-baff-ac5a6ad0c77f"}]: dispatch 2024-09-13T00:42:55.517 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:55 smithi077 bash[80681]: audit 2024-09-13T00:42:54.247126+0000 mon.a (mon.0) 517 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/88d10567-eb59-49b6-baff-ac5a6ad0c77f"}]: dispatch 2024-09-13T00:42:55.517 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:55 smithi077 bash[80681]: audit 2024-09-13T00:42:54.250697+0000 mon.a (mon.0) 518 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/88d10567-eb59-49b6-baff-ac5a6ad0c77f"}]': finished 2024-09-13T00:42:55.517 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:55 smithi077 bash[80681]: cluster 2024-09-13T00:42:54.417145+0000 mgr.y (mgr.25304) 184 : cluster [DBG] pgmap v310: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T00:42:55.517 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:55 smithi077 bash[83127]: cluster 2024-09-13T00:42:54.172534+0000 mon.a (mon.0) 516 : cluster [DBG] osdmap e479: 8 total, 8 up, 8 in 2024-09-13T00:42:55.517 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:55 smithi077 bash[83127]: audit 2024-09-13T00:42:54.246716+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/88d10567-eb59-49b6-baff-ac5a6ad0c77f"}]: dispatch 2024-09-13T00:42:55.518 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:55 smithi077 bash[83127]: audit 2024-09-13T00:42:54.247126+0000 mon.a (mon.0) 517 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/88d10567-eb59-49b6-baff-ac5a6ad0c77f"}]: dispatch 2024-09-13T00:42:55.518 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:55 smithi077 bash[83127]: audit 2024-09-13T00:42:54.250697+0000 mon.a (mon.0) 518 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/88d10567-eb59-49b6-baff-ac5a6ad0c77f"}]': finished 2024-09-13T00:42:55.518 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:55 smithi077 bash[83127]: cluster 2024-09-13T00:42:54.417145+0000 mgr.y (mgr.25304) 184 : cluster [DBG] pgmap v310: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T00:42:55.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:55 smithi088 bash[68960]: cluster 2024-09-13T00:42:54.172534+0000 mon.a (mon.0) 516 : cluster [DBG] osdmap e479: 8 total, 8 up, 8 in 2024-09-13T00:42:55.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:55 smithi088 bash[68960]: audit 2024-09-13T00:42:54.246716+0000 mon.c (mon.2) 16 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/88d10567-eb59-49b6-baff-ac5a6ad0c77f"}]: dispatch 2024-09-13T00:42:55.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:55 smithi088 bash[68960]: audit 2024-09-13T00:42:54.247126+0000 mon.a (mon.0) 517 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/88d10567-eb59-49b6-baff-ac5a6ad0c77f"}]: dispatch 2024-09-13T00:42:55.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:55 smithi088 bash[68960]: audit 2024-09-13T00:42:54.250697+0000 mon.a (mon.0) 518 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/88d10567-eb59-49b6-baff-ac5a6ad0c77f"}]': finished 2024-09-13T00:42:55.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:55 smithi088 bash[68960]: cluster 2024-09-13T00:42:54.417145+0000 mgr.y (mgr.25304) 184 : cluster [DBG] pgmap v310: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T00:42:55.712 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-09-13T00:42:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:56 smithi077 bash[80681]: cluster 2024-09-13T00:42:55.272519+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e480: 8 total, 8 up, 8 in 2024-09-13T00:42:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:56 smithi077 bash[80681]: cluster 2024-09-13T00:42:55.901867+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e481: 8 total, 8 up, 8 in 2024-09-13T00:42:56.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:56 smithi077 bash[83127]: cluster 2024-09-13T00:42:55.272519+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e480: 8 total, 8 up, 8 in 2024-09-13T00:42:56.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:56 smithi077 bash[83127]: cluster 2024-09-13T00:42:55.901867+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e481: 8 total, 8 up, 8 in 2024-09-13T00:42:56.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:56 smithi088 bash[68960]: cluster 2024-09-13T00:42:55.272519+0000 mon.a (mon.0) 519 : cluster [DBG] osdmap e480: 8 total, 8 up, 8 in 2024-09-13T00:42:56.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:56 smithi088 bash[68960]: cluster 2024-09-13T00:42:55.901867+0000 mon.a (mon.0) 520 : cluster [DBG] osdmap e481: 8 total, 8 up, 8 in 2024-09-13T00:42:58.140 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:57 smithi077 bash[83127]: cluster 2024-09-13T00:42:56.417860+0000 mgr.y (mgr.25304) 185 : cluster [DBG] pgmap v313: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T00:42:58.140 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:57 smithi077 bash[83127]: cluster 2024-09-13T00:42:56.903736+0000 mon.a (mon.0) 521 : cluster [DBG] osdmap e482: 8 total, 8 up, 8 in 2024-09-13T00:42:58.141 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:57 smithi077 bash[80681]: cluster 2024-09-13T00:42:56.417860+0000 mgr.y (mgr.25304) 185 : cluster [DBG] pgmap v313: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T00:42:58.141 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:57 smithi077 bash[80681]: cluster 2024-09-13T00:42:56.903736+0000 mon.a (mon.0) 521 : cluster [DBG] osdmap e482: 8 total, 8 up, 8 in 2024-09-13T00:42:58.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:57 smithi088 bash[68960]: cluster 2024-09-13T00:42:56.417860+0000 mgr.y (mgr.25304) 185 : cluster [DBG] pgmap v313: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-13T00:42:58.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:57 smithi088 bash[68960]: cluster 2024-09-13T00:42:56.903736+0000 mon.a (mon.0) 521 : cluster [DBG] osdmap e482: 8 total, 8 up, 8 in 2024-09-13T00:42:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:42:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:42:58] "GET /metrics HTTP/1.1" 200 36431 "" "Prometheus/2.43.0" 2024-09-13T00:42:59.214 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:58 smithi077 bash[80681]: cluster 2024-09-13T00:42:57.904775+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e483: 8 total, 8 up, 8 in 2024-09-13T00:42:59.215 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:58 smithi077 bash[80681]: audit 2024-09-13T00:42:58.414067+0000 mon.a (mon.0) 523 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:59.215 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:58 smithi077 bash[83127]: cluster 2024-09-13T00:42:57.904775+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e483: 8 total, 8 up, 8 in 2024-09-13T00:42:59.215 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:58 smithi077 bash[83127]: audit 2024-09-13T00:42:58.414067+0000 mon.a (mon.0) 523 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:58 smithi088 bash[68960]: cluster 2024-09-13T00:42:57.904775+0000 mon.a (mon.0) 522 : cluster [DBG] osdmap e483: 8 total, 8 up, 8 in 2024-09-13T00:42:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:58 smithi088 bash[68960]: audit 2024-09-13T00:42:58.414067+0000 mon.a (mon.0) 523 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:42:59.487 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:59 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:42:59.488 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:42:59 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:42:59.488 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:42:59 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:42:59.488 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:42:59 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:42:59.489 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:42:59 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:42:59.489 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:42:59 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:42:59.489 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:42:59 smithi077 systemd[1]: Stopping Ceph osd.0 for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:42:59.489 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:42:59 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:42:59.490 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:59 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:42:59.490 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:42:59 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:42:59.778 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:42:59 smithi077 bash[35108]: debug 2024-09-13T00:42:59.611+0000 7efcd1f08640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:42:59.779 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:42:59 smithi077 bash[35108]: debug 2024-09-13T00:42:59.611+0000 7efcd1f08640 -1 osd.0 484 *** Got signal Terminated *** 2024-09-13T00:42:59.779 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:42:59 smithi077 bash[35108]: debug 2024-09-13T00:42:59.611+0000 7efcd1f08640 -1 osd.0 484 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T00:43:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:59 smithi077 bash[80681]: debug 2024-09-13T00:42:59.915+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:43:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:59 smithi077 bash[80681]: cluster 2024-09-13T00:42:58.422597+0000 mgr.y (mgr.25304) 186 : cluster [DBG] pgmap v316: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:43:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:59 smithi077 bash[80681]: cluster 2024-09-13T00:42:58.931343+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e484: 8 total, 8 up, 8 in 2024-09-13T00:43:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:42:59 smithi077 bash[80681]: cluster 2024-09-13T00:42:59.617670+0000 mon.a (mon.0) 525 : cluster [INF] osd.0 marked itself down and dead 2024-09-13T00:43:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:59 smithi077 bash[83127]: cluster 2024-09-13T00:42:58.422597+0000 mgr.y (mgr.25304) 186 : cluster [DBG] pgmap v316: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:43:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:59 smithi077 bash[83127]: cluster 2024-09-13T00:42:58.931343+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e484: 8 total, 8 up, 8 in 2024-09-13T00:43:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:42:59 smithi077 bash[83127]: cluster 2024-09-13T00:42:59.617670+0000 mon.a (mon.0) 525 : cluster [INF] osd.0 marked itself down and dead 2024-09-13T00:43:00.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:59 smithi088 bash[68960]: cluster 2024-09-13T00:42:58.422597+0000 mgr.y (mgr.25304) 186 : cluster [DBG] pgmap v316: 105 pgs: 105 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:43:00.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:59 smithi088 bash[68960]: cluster 2024-09-13T00:42:58.931343+0000 mon.a (mon.0) 524 : cluster [DBG] osdmap e484: 8 total, 8 up, 8 in 2024-09-13T00:43:00.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:42:59 smithi088 bash[68960]: cluster 2024-09-13T00:42:59.617670+0000 mon.a (mon.0) 525 : cluster [INF] osd.0 marked itself down and dead 2024-09-13T00:43:00.499 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:43:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:00 smithi077 bash[80681]: cluster 2024-09-13T00:42:59.920688+0000 mon.a (mon.0) 526 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T00:43:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:00 smithi077 bash[80681]: cluster 2024-09-13T00:42:59.920752+0000 mon.a (mon.0) 527 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:43:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:00 smithi077 bash[80681]: cluster 2024-09-13T00:42:59.933504+0000 mon.a (mon.0) 528 : cluster [DBG] osdmap e485: 8 total, 7 up, 8 in 2024-09-13T00:43:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:00 smithi077 bash[80681]: cluster 2024-09-13T00:43:00.933200+0000 mon.a (mon.0) 529 : cluster [DBG] osdmap e486: 8 total, 7 up, 8 in 2024-09-13T00:43:01.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:00 smithi077 bash[83127]: cluster 2024-09-13T00:42:59.920688+0000 mon.a (mon.0) 526 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T00:43:01.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:00 smithi077 bash[83127]: cluster 2024-09-13T00:42:59.920752+0000 mon.a (mon.0) 527 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:43:01.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:00 smithi077 bash[83127]: cluster 2024-09-13T00:42:59.933504+0000 mon.a (mon.0) 528 : cluster [DBG] osdmap e485: 8 total, 7 up, 8 in 2024-09-13T00:43:01.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:00 smithi077 bash[83127]: cluster 2024-09-13T00:43:00.933200+0000 mon.a (mon.0) 529 : cluster [DBG] osdmap e486: 8 total, 7 up, 8 in 2024-09-13T00:43:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:00 smithi088 bash[68960]: cluster 2024-09-13T00:42:59.920688+0000 mon.a (mon.0) 526 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T00:43:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:00 smithi088 bash[68960]: cluster 2024-09-13T00:42:59.920752+0000 mon.a (mon.0) 527 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:43:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:00 smithi088 bash[68960]: cluster 2024-09-13T00:42:59.933504+0000 mon.a (mon.0) 528 : cluster [DBG] osdmap e485: 8 total, 7 up, 8 in 2024-09-13T00:43:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:00 smithi088 bash[68960]: cluster 2024-09-13T00:43:00.933200+0000 mon.a (mon.0) 529 : cluster [DBG] osdmap e486: 8 total, 7 up, 8 in 2024-09-13T00:43:01.985 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:01 smithi077 bash[89136]: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-0 2024-09-13T00:43:02.278 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:02 smithi077 bash[89214]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-0 2024-09-13T00:43:02.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:01 smithi077 bash[80681]: cluster 2024-09-13T00:43:00.423288+0000 mgr.y (mgr.25304) 187 : cluster [DBG] pgmap v319: 105 pgs: 13 stale+active+clean, 92 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T00:43:02.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:01 smithi077 bash[83127]: cluster 2024-09-13T00:43:00.423288+0000 mgr.y (mgr.25304) 187 : cluster [DBG] pgmap v319: 105 pgs: 13 stale+active+clean, 92 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T00:43:02.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:01 smithi088 bash[68960]: cluster 2024-09-13T00:43:00.423288+0000 mgr.y (mgr.25304) 187 : cluster [DBG] pgmap v319: 105 pgs: 13 stale+active+clean, 92 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T00:43:02.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:43:02 smithi088 bash[61791]: ts=2024-09-13T00:43:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:43:03.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:02 smithi077 bash[80681]: cluster 2024-09-13T00:43:01.971399+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e487: 8 total, 7 up, 8 in 2024-09-13T00:43:03.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:02 smithi077 bash[83127]: cluster 2024-09-13T00:43:01.971399+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e487: 8 total, 7 up, 8 in 2024-09-13T00:43:03.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:02 smithi088 bash[68960]: cluster 2024-09-13T00:43:01.971399+0000 mon.a (mon.0) 530 : cluster [DBG] osdmap e487: 8 total, 7 up, 8 in 2024-09-13T00:43:04.208 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:03 smithi077 bash[83127]: cluster 2024-09-13T00:43:02.424034+0000 mgr.y (mgr.25304) 188 : cluster [DBG] pgmap v322: 105 pgs: 13 stale+active+clean, 92 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T00:43:04.209 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:03 smithi077 bash[83127]: cluster 2024-09-13T00:43:02.992294+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e488: 8 total, 7 up, 8 in 2024-09-13T00:43:04.209 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:03 smithi077 bash[80681]: cluster 2024-09-13T00:43:02.424034+0000 mgr.y (mgr.25304) 188 : cluster [DBG] pgmap v322: 105 pgs: 13 stale+active+clean, 92 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T00:43:04.209 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:03 smithi077 bash[80681]: cluster 2024-09-13T00:43:02.992294+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e488: 8 total, 7 up, 8 in 2024-09-13T00:43:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:03 smithi088 bash[68960]: cluster 2024-09-13T00:43:02.424034+0000 mgr.y (mgr.25304) 188 : cluster [DBG] pgmap v322: 105 pgs: 13 stale+active+clean, 92 active+clean; 583 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-13T00:43:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:03 smithi088 bash[68960]: cluster 2024-09-13T00:43:02.992294+0000 mon.a (mon.0) 531 : cluster [DBG] osdmap e488: 8 total, 7 up, 8 in 2024-09-13T00:43:05.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:05 smithi077 bash[80681]: cluster 2024-09-13T00:43:04.001290+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e489: 8 total, 7 up, 8 in 2024-09-13T00:43:05.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:05 smithi077 bash[83127]: cluster 2024-09-13T00:43:04.001290+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e489: 8 total, 7 up, 8 in 2024-09-13T00:43:05.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:05 smithi088 bash[68960]: cluster 2024-09-13T00:43:04.001290+0000 mon.a (mon.0) 532 : cluster [DBG] osdmap e489: 8 total, 7 up, 8 in 2024-09-13T00:43:06.330 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:06 smithi077 bash[80681]: cluster 2024-09-13T00:43:04.425538+0000 mgr.y (mgr.25304) 189 : cluster [DBG] pgmap v325: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 93 op/s; 46/447 objects degraded (10.291%) 2024-09-13T00:43:06.331 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:06 smithi077 bash[80681]: cluster 2024-09-13T00:43:04.993938+0000 mon.a (mon.0) 533 : cluster [WRN] Health check failed: Degraded data redundancy: 46/447 objects degraded (10.291%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:06.331 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:06 smithi077 bash[80681]: cluster 2024-09-13T00:43:05.006600+0000 mon.a (mon.0) 534 : cluster [DBG] osdmap e490: 8 total, 7 up, 8 in 2024-09-13T00:43:06.331 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:06 smithi077 bash[83127]: cluster 2024-09-13T00:43:04.425538+0000 mgr.y (mgr.25304) 189 : cluster [DBG] pgmap v325: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 93 op/s; 46/447 objects degraded (10.291%) 2024-09-13T00:43:06.331 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:06 smithi077 bash[83127]: cluster 2024-09-13T00:43:04.993938+0000 mon.a (mon.0) 533 : cluster [WRN] Health check failed: Degraded data redundancy: 46/447 objects degraded (10.291%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:06.331 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:06 smithi077 bash[83127]: cluster 2024-09-13T00:43:05.006600+0000 mon.a (mon.0) 534 : cluster [DBG] osdmap e490: 8 total, 7 up, 8 in 2024-09-13T00:43:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:06 smithi088 bash[68960]: cluster 2024-09-13T00:43:04.425538+0000 mgr.y (mgr.25304) 189 : cluster [DBG] pgmap v325: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 93 op/s; 46/447 objects degraded (10.291%) 2024-09-13T00:43:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:06 smithi088 bash[68960]: cluster 2024-09-13T00:43:04.993938+0000 mon.a (mon.0) 533 : cluster [WRN] Health check failed: Degraded data redundancy: 46/447 objects degraded (10.291%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:06 smithi088 bash[68960]: cluster 2024-09-13T00:43:05.006600+0000 mon.a (mon.0) 534 : cluster [DBG] osdmap e490: 8 total, 7 up, 8 in 2024-09-13T00:43:06.684 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:43:06 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:06.684 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:43:06 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:06.684 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:43:06 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:06.685 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:06 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-13T00:43:06.685 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:06 smithi077 systemd[1]: Stopped Ceph osd.0 for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:43:06.685 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:06 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:06.685 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:06 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:06.685 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:06 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:06.686 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:43:06 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:06.686 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:06 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:06.686 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:43:06 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:07.013 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:06 smithi077 systemd[1]: Started Ceph osd.0 for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:43:07.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:07 smithi077 bash[80681]: cluster 2024-09-13T00:43:06.024348+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e491: 8 total, 7 up, 8 in 2024-09-13T00:43:07.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:07 smithi077 bash[80681]: audit 2024-09-13T00:43:06.726949+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:07.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:07 smithi077 bash[80681]: audit 2024-09-13T00:43:06.733820+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:07.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:07 smithi077 bash[80681]: audit 2024-09-13T00:43:06.734898+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:43:07.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:07 smithi077 bash[83127]: cluster 2024-09-13T00:43:06.024348+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e491: 8 total, 7 up, 8 in 2024-09-13T00:43:07.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:07 smithi077 bash[83127]: audit 2024-09-13T00:43:06.726949+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:07.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:07 smithi077 bash[83127]: audit 2024-09-13T00:43:06.733820+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:07.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:07 smithi077 bash[83127]: audit 2024-09-13T00:43:06.734898+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:43:07.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:07 smithi088 bash[68960]: cluster 2024-09-13T00:43:06.024348+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e491: 8 total, 7 up, 8 in 2024-09-13T00:43:07.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:07 smithi088 bash[68960]: audit 2024-09-13T00:43:06.726949+0000 mon.a (mon.0) 536 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:07.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:07 smithi088 bash[68960]: audit 2024-09-13T00:43:06.733820+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:07.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:07 smithi088 bash[68960]: audit 2024-09-13T00:43:06.734898+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:43:08.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:08 smithi077 bash[83127]: cluster 2024-09-13T00:43:06.426377+0000 mgr.y (mgr.25304) 190 : cluster [DBG] pgmap v328: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 93 op/s; 46/447 objects degraded (10.291%) 2024-09-13T00:43:08.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:08 smithi077 bash[83127]: cluster 2024-09-13T00:43:07.027250+0000 mon.a (mon.0) 539 : cluster [DBG] osdmap e492: 8 total, 7 up, 8 in 2024-09-13T00:43:08.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:08 smithi077 bash[80681]: cluster 2024-09-13T00:43:06.426377+0000 mgr.y (mgr.25304) 190 : cluster [DBG] pgmap v328: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 93 op/s; 46/447 objects degraded (10.291%) 2024-09-13T00:43:08.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:08 smithi077 bash[80681]: cluster 2024-09-13T00:43:07.027250+0000 mon.a (mon.0) 539 : cluster [DBG] osdmap e492: 8 total, 7 up, 8 in 2024-09-13T00:43:08.188 INFO:teuthology.orchestra.run.smithi077.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-13T00:43:08.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:08 smithi088 bash[68960]: cluster 2024-09-13T00:43:06.426377+0000 mgr.y (mgr.25304) 190 : cluster [DBG] pgmap v328: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 11 KiB/s wr, 93 op/s; 46/447 objects degraded (10.291%) 2024-09-13T00:43:08.354 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:08 smithi088 bash[68960]: cluster 2024-09-13T00:43:07.027250+0000 mon.a (mon.0) 539 : cluster [DBG] osdmap e492: 8 total, 7 up, 8 in 2024-09-13T00:43:08.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:43:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:43:08] "GET /metrics HTTP/1.1" 200 36431 "" "Prometheus/2.43.0" 2024-09-13T00:43:09.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:09 smithi077 bash[80681]: cluster 2024-09-13T00:43:08.037918+0000 mon.a (mon.0) 540 : cluster [DBG] osdmap e493: 8 total, 7 up, 8 in 2024-09-13T00:43:09.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:09 smithi077 bash[80681]: audit 2024-09-13T00:43:08.179623+0000 mon.a (mon.0) 541 : audit [DBG] from='client.? 172.21.15.77:0/3960237745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:43:09.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:09 smithi077 bash[83127]: cluster 2024-09-13T00:43:08.037918+0000 mon.a (mon.0) 540 : cluster [DBG] osdmap e493: 8 total, 7 up, 8 in 2024-09-13T00:43:09.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:09 smithi077 bash[83127]: audit 2024-09-13T00:43:08.179623+0000 mon.a (mon.0) 541 : audit [DBG] from='client.? 172.21.15.77:0/3960237745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:43:09.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:09 smithi088 bash[68960]: cluster 2024-09-13T00:43:08.037918+0000 mon.a (mon.0) 540 : cluster [DBG] osdmap e493: 8 total, 7 up, 8 in 2024-09-13T00:43:09.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:09 smithi088 bash[68960]: audit 2024-09-13T00:43:08.179623+0000 mon.a (mon.0) 541 : audit [DBG] from='client.? 172.21.15.77:0/3960237745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-09-13T00:43:09.511 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph orch ps' 2024-09-13T00:43:10.141 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:10 smithi077 bash[83127]: cluster 2024-09-13T00:43:08.427200+0000 mgr.y (mgr.25304) 191 : cluster [DBG] pgmap v331: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 46/447 objects degraded (10.291%) 2024-09-13T00:43:10.141 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:10 smithi077 bash[83127]: cluster 2024-09-13T00:43:09.049195+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e494: 8 total, 7 up, 8 in 2024-09-13T00:43:10.141 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:09 smithi077 bash[89432]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T00:43:10.141 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:09 smithi077 bash[89432]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T00:43:10.141 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:09 smithi077 bash[89432]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T00:43:10.141 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:09 smithi077 bash[89432]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T00:43:10.141 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:09 smithi077 bash[89432]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-13T00:43:10.141 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:09 smithi077 bash[89432]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-13T00:43:10.142 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:09 smithi077 bash[89432]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-13T00:43:10.142 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:10 smithi077 bash[80681]: cluster 2024-09-13T00:43:08.427200+0000 mgr.y (mgr.25304) 191 : cluster [DBG] pgmap v331: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 46/447 objects degraded (10.291%) 2024-09-13T00:43:10.142 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:10 smithi077 bash[80681]: cluster 2024-09-13T00:43:09.049195+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e494: 8 total, 7 up, 8 in 2024-09-13T00:43:10.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:10 smithi088 bash[68960]: cluster 2024-09-13T00:43:08.427200+0000 mgr.y (mgr.25304) 191 : cluster [DBG] pgmap v331: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 46/447 objects degraded (10.291%) 2024-09-13T00:43:10.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:10 smithi088 bash[68960]: cluster 2024-09-13T00:43:09.049195+0000 mon.a (mon.0) 542 : cluster [DBG] osdmap e494: 8 total, 7 up, 8 in 2024-09-13T00:43:11.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:11 smithi077 bash[80681]: cluster 2024-09-13T00:43:10.047682+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e495: 8 total, 7 up, 8 in 2024-09-13T00:43:11.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:11 smithi077 bash[83127]: cluster 2024-09-13T00:43:10.047682+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e495: 8 total, 7 up, 8 in 2024-09-13T00:43:11.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:11 smithi088 bash[68960]: cluster 2024-09-13T00:43:10.047682+0000 mon.a (mon.0) 543 : cluster [DBG] osdmap e495: 8 total, 7 up, 8 in 2024-09-13T00:43:12.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:12 smithi088 bash[68960]: cluster 2024-09-13T00:43:10.428130+0000 mgr.y (mgr.25304) 192 : cluster [DBG] pgmap v334: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 173 KiB/s rd, 20 KiB/s wr, 234 op/s; 46/480 objects degraded (9.583%) 2024-09-13T00:43:12.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:12 smithi088 bash[68960]: cluster 2024-09-13T00:43:11.056965+0000 mon.a (mon.0) 544 : cluster [WRN] Health check update: Degraded data redundancy: 46/480 objects degraded (9.583%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:12.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:12 smithi088 bash[68960]: cluster 2024-09-13T00:43:11.074815+0000 mon.a (mon.0) 545 : cluster [DBG] osdmap e496: 8 total, 7 up, 8 in 2024-09-13T00:43:12.528 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:12 smithi077 bash[89778]: debug 2024-09-13T00:43:12.251+0000 7fd06e34a640 1 -- 172.21.15.77:0/1516051525 <== mon.0 v2:172.21.15.77:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5569dd1071e0 con 0x5569dd0c7c00 2024-09-13T00:43:12.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:12 smithi077 bash[83127]: cluster 2024-09-13T00:43:10.428130+0000 mgr.y (mgr.25304) 192 : cluster [DBG] pgmap v334: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 173 KiB/s rd, 20 KiB/s wr, 234 op/s; 46/480 objects degraded (9.583%) 2024-09-13T00:43:12.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:12 smithi077 bash[83127]: cluster 2024-09-13T00:43:11.056965+0000 mon.a (mon.0) 544 : cluster [WRN] Health check update: Degraded data redundancy: 46/480 objects degraded (9.583%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:12.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:12 smithi077 bash[83127]: cluster 2024-09-13T00:43:11.074815+0000 mon.a (mon.0) 545 : cluster [DBG] osdmap e496: 8 total, 7 up, 8 in 2024-09-13T00:43:12.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:12 smithi077 bash[80681]: cluster 2024-09-13T00:43:10.428130+0000 mgr.y (mgr.25304) 192 : cluster [DBG] pgmap v334: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 173 KiB/s rd, 20 KiB/s wr, 234 op/s; 46/480 objects degraded (9.583%) 2024-09-13T00:43:12.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:12 smithi077 bash[80681]: cluster 2024-09-13T00:43:11.056965+0000 mon.a (mon.0) 544 : cluster [WRN] Health check update: Degraded data redundancy: 46/480 objects degraded (9.583%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:12.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:12 smithi077 bash[80681]: cluster 2024-09-13T00:43:11.074815+0000 mon.a (mon.0) 545 : cluster [DBG] osdmap e496: 8 total, 7 up, 8 in 2024-09-13T00:43:12.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:43:12 smithi088 bash[61791]: ts=2024-09-13T00:43:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:43:13.278 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:12 smithi077 bash[89778]: debug 2024-09-13T00:43:12.839+0000 7fd070bb0640 -1 Falling back to public interface 2024-09-13T00:43:13.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:13 smithi077 bash[80681]: cluster 2024-09-13T00:43:12.078478+0000 mon.a (mon.0) 546 : cluster [DBG] osdmap e497: 8 total, 7 up, 8 in 2024-09-13T00:43:13.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:13 smithi077 bash[80681]: audit 2024-09-13T00:43:12.966803+0000 mon.a (mon.0) 547 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:13.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:13 smithi077 bash[80681]: audit 2024-09-13T00:43:12.972091+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:13.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:13 smithi077 bash[83127]: cluster 2024-09-13T00:43:12.078478+0000 mon.a (mon.0) 546 : cluster [DBG] osdmap e497: 8 total, 7 up, 8 in 2024-09-13T00:43:13.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:13 smithi077 bash[83127]: audit 2024-09-13T00:43:12.966803+0000 mon.a (mon.0) 547 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:13.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:13 smithi077 bash[83127]: audit 2024-09-13T00:43:12.972091+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:13.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:13 smithi088 bash[68960]: cluster 2024-09-13T00:43:12.078478+0000 mon.a (mon.0) 546 : cluster [DBG] osdmap e497: 8 total, 7 up, 8 in 2024-09-13T00:43:13.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:13 smithi088 bash[68960]: audit 2024-09-13T00:43:12.966803+0000 mon.a (mon.0) 547 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:13.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:13 smithi088 bash[68960]: audit 2024-09-13T00:43:12.972091+0000 mon.a (mon.0) 548 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:14.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:14 smithi088 bash[68960]: cluster 2024-09-13T00:43:12.428616+0000 mgr.y (mgr.25304) 193 : cluster [DBG] pgmap v337: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 173 KiB/s rd, 20 KiB/s wr, 234 op/s; 46/480 objects degraded (9.583%) 2024-09-13T00:43:14.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:14 smithi088 bash[68960]: cluster 2024-09-13T00:43:13.088041+0000 mon.a (mon.0) 549 : cluster [DBG] osdmap e498: 8 total, 7 up, 8 in 2024-09-13T00:43:14.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:14 smithi088 bash[68960]: audit 2024-09-13T00:43:13.417917+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:14.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:14 smithi088 bash[68960]: audit 2024-09-13T00:43:13.419438+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:14.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:14 smithi088 bash[68960]: audit 2024-09-13T00:43:13.657834+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:14.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:14 smithi088 bash[68960]: audit 2024-09-13T00:43:13.666579+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:14.361 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:43:14.459 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:14 smithi077 bash[83127]: cluster 2024-09-13T00:43:12.428616+0000 mgr.y (mgr.25304) 193 : cluster [DBG] pgmap v337: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 173 KiB/s rd, 20 KiB/s wr, 234 op/s; 46/480 objects degraded (9.583%) 2024-09-13T00:43:14.459 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:14 smithi077 bash[83127]: cluster 2024-09-13T00:43:13.088041+0000 mon.a (mon.0) 549 : cluster [DBG] osdmap e498: 8 total, 7 up, 8 in 2024-09-13T00:43:14.459 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:14 smithi077 bash[83127]: audit 2024-09-13T00:43:13.417917+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:14.459 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:14 smithi077 bash[83127]: audit 2024-09-13T00:43:13.419438+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:14.459 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:14 smithi077 bash[83127]: audit 2024-09-13T00:43:13.657834+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:14.459 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:14 smithi077 bash[83127]: audit 2024-09-13T00:43:13.666579+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:14.459 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:14 smithi077 bash[80681]: cluster 2024-09-13T00:43:12.428616+0000 mgr.y (mgr.25304) 193 : cluster [DBG] pgmap v337: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 173 KiB/s rd, 20 KiB/s wr, 234 op/s; 46/480 objects degraded (9.583%) 2024-09-13T00:43:14.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:14 smithi077 bash[80681]: cluster 2024-09-13T00:43:13.088041+0000 mon.a (mon.0) 549 : cluster [DBG] osdmap e498: 8 total, 7 up, 8 in 2024-09-13T00:43:14.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:14 smithi077 bash[80681]: audit 2024-09-13T00:43:13.417917+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:14.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:14 smithi077 bash[80681]: audit 2024-09-13T00:43:13.419438+0000 mon.a (mon.0) 551 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:14.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:14 smithi077 bash[80681]: audit 2024-09-13T00:43:13.657834+0000 mon.a (mon.0) 552 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:14.460 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:14 smithi077 bash[80681]: audit 2024-09-13T00:43:13.666579+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:15.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:15 smithi077 bash[83127]: cluster 2024-09-13T00:43:14.086201+0000 mon.a (mon.0) 554 : cluster [DBG] osdmap e499: 8 total, 7 up, 8 in 2024-09-13T00:43:15.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:15 smithi077 bash[80681]: cluster 2024-09-13T00:43:14.086201+0000 mon.a (mon.0) 554 : cluster [DBG] osdmap e499: 8 total, 7 up, 8 in 2024-09-13T00:43:15.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:15 smithi088 bash[68960]: cluster 2024-09-13T00:43:14.086201+0000 mon.a (mon.0) 554 : cluster [DBG] osdmap e499: 8 total, 7 up, 8 in 2024-09-13T00:43:16.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:16 smithi077 bash[80681]: cluster 2024-09-13T00:43:14.429408+0000 mgr.y (mgr.25304) 194 : cluster [DBG] pgmap v340: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s; 41/423 objects degraded (9.693%) 2024-09-13T00:43:16.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:16 smithi077 bash[80681]: cluster 2024-09-13T00:43:15.115237+0000 mon.a (mon.0) 555 : cluster [DBG] osdmap e500: 8 total, 7 up, 8 in 2024-09-13T00:43:16.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:16 smithi077 bash[80681]: audit 2024-09-13T00:43:15.936791+0000 mon.a (mon.0) 556 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/3174548933,v1:172.21.15.77:6803/3174548933]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T00:43:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:16 smithi077 bash[83127]: cluster 2024-09-13T00:43:14.429408+0000 mgr.y (mgr.25304) 194 : cluster [DBG] pgmap v340: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s; 41/423 objects degraded (9.693%) 2024-09-13T00:43:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:16 smithi077 bash[83127]: cluster 2024-09-13T00:43:15.115237+0000 mon.a (mon.0) 555 : cluster [DBG] osdmap e500: 8 total, 7 up, 8 in 2024-09-13T00:43:16.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:16 smithi077 bash[83127]: audit 2024-09-13T00:43:15.936791+0000 mon.a (mon.0) 556 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/3174548933,v1:172.21.15.77:6803/3174548933]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T00:43:16.279 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:15 smithi077 bash[89778]: debug 2024-09-13T00:43:15.927+0000 7fd070bb0640 -1 osd.0 484 log_to_monitors true 2024-09-13T00:43:16.279 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:16 smithi077 bash[89778]: debug 2024-09-13T00:43:16.151+0000 7fd06895a640 -1 osd.0 484 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T00:43:16.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:16 smithi088 bash[68960]: cluster 2024-09-13T00:43:14.429408+0000 mgr.y (mgr.25304) 194 : cluster [DBG] pgmap v340: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s; 41/423 objects degraded (9.693%) 2024-09-13T00:43:16.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:16 smithi088 bash[68960]: cluster 2024-09-13T00:43:15.115237+0000 mon.a (mon.0) 555 : cluster [DBG] osdmap e500: 8 total, 7 up, 8 in 2024-09-13T00:43:16.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:16 smithi088 bash[68960]: audit 2024-09-13T00:43:15.936791+0000 mon.a (mon.0) 556 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/3174548933,v1:172.21.15.77:6803/3174548933]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-13T00:43:17.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:17 smithi077 bash[83127]: audit 2024-09-13T00:43:16.116445+0000 mon.a (mon.0) 557 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/3174548933,v1:172.21.15.77:6803/3174548933]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T00:43:17.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:17 smithi077 bash[83127]: cluster 2024-09-13T00:43:16.116652+0000 mon.a (mon.0) 558 : cluster [DBG] osdmap e501: 8 total, 7 up, 8 in 2024-09-13T00:43:17.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:17 smithi077 bash[83127]: audit 2024-09-13T00:43:16.117351+0000 mon.a (mon.0) 559 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/3174548933,v1:172.21.15.77:6803/3174548933]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:43:17.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:17 smithi077 bash[80681]: debug 2024-09-13T00:43:17.107+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:43:17.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:17 smithi077 bash[80681]: audit 2024-09-13T00:43:16.116445+0000 mon.a (mon.0) 557 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/3174548933,v1:172.21.15.77:6803/3174548933]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T00:43:17.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:17 smithi077 bash[80681]: cluster 2024-09-13T00:43:16.116652+0000 mon.a (mon.0) 558 : cluster [DBG] osdmap e501: 8 total, 7 up, 8 in 2024-09-13T00:43:17.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:17 smithi077 bash[80681]: audit 2024-09-13T00:43:16.117351+0000 mon.a (mon.0) 559 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/3174548933,v1:172.21.15.77:6803/3174548933]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:43:17.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:17 smithi088 bash[68960]: audit 2024-09-13T00:43:16.116445+0000 mon.a (mon.0) 557 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/3174548933,v1:172.21.15.77:6803/3174548933]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-13T00:43:17.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:17 smithi088 bash[68960]: cluster 2024-09-13T00:43:16.116652+0000 mon.a (mon.0) 558 : cluster [DBG] osdmap e501: 8 total, 7 up, 8 in 2024-09-13T00:43:17.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:17 smithi088 bash[68960]: audit 2024-09-13T00:43:16.117351+0000 mon.a (mon.0) 559 : audit [INF] from='osd.0 [v2:172.21.15.77:6802/3174548933,v1:172.21.15.77:6803/3174548933]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:43:18.451 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:18 smithi077 bash[80681]: cluster 2024-09-13T00:43:16.430057+0000 mgr.y (mgr.25304) 195 : cluster [DBG] pgmap v343: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s; 41/423 objects degraded (9.693%) 2024-09-13T00:43:18.451 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:18 smithi077 bash[80681]: cluster 2024-09-13T00:43:17.114047+0000 mon.a (mon.0) 560 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:43:18.451 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:18 smithi077 bash[80681]: cluster 2024-09-13T00:43:17.114073+0000 mon.a (mon.0) 561 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T00:43:18.452 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:18 smithi077 bash[80681]: cluster 2024-09-13T00:43:17.128313+0000 mon.a (mon.0) 562 : cluster [INF] osd.0 [v2:172.21.15.77:6802/3174548933,v1:172.21.15.77:6803/3174548933] boot 2024-09-13T00:43:18.452 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:18 smithi077 bash[80681]: cluster 2024-09-13T00:43:17.128463+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e502: 8 total, 8 up, 8 in 2024-09-13T00:43:18.452 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:18 smithi077 bash[80681]: audit 2024-09-13T00:43:17.129597+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:43:18.452 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:18 smithi077 bash[83127]: cluster 2024-09-13T00:43:16.430057+0000 mgr.y (mgr.25304) 195 : cluster [DBG] pgmap v343: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s; 41/423 objects degraded (9.693%) 2024-09-13T00:43:18.452 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:18 smithi077 bash[83127]: cluster 2024-09-13T00:43:17.114047+0000 mon.a (mon.0) 560 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:43:18.452 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:18 smithi077 bash[83127]: cluster 2024-09-13T00:43:17.114073+0000 mon.a (mon.0) 561 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T00:43:18.452 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:18 smithi077 bash[83127]: cluster 2024-09-13T00:43:17.128313+0000 mon.a (mon.0) 562 : cluster [INF] osd.0 [v2:172.21.15.77:6802/3174548933,v1:172.21.15.77:6803/3174548933] boot 2024-09-13T00:43:18.452 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:18 smithi077 bash[83127]: cluster 2024-09-13T00:43:17.128463+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e502: 8 total, 8 up, 8 in 2024-09-13T00:43:18.452 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:18 smithi077 bash[83127]: audit 2024-09-13T00:43:17.129597+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:43:18.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:18 smithi088 bash[68960]: cluster 2024-09-13T00:43:16.430057+0000 mgr.y (mgr.25304) 195 : cluster [DBG] pgmap v343: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s; 41/423 objects degraded (9.693%) 2024-09-13T00:43:18.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:18 smithi088 bash[68960]: cluster 2024-09-13T00:43:17.114047+0000 mon.a (mon.0) 560 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:43:18.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:18 smithi088 bash[68960]: cluster 2024-09-13T00:43:17.114073+0000 mon.a (mon.0) 561 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T00:43:18.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:18 smithi088 bash[68960]: cluster 2024-09-13T00:43:17.128313+0000 mon.a (mon.0) 562 : cluster [INF] osd.0 [v2:172.21.15.77:6802/3174548933,v1:172.21.15.77:6803/3174548933] boot 2024-09-13T00:43:18.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:18 smithi088 bash[68960]: cluster 2024-09-13T00:43:17.128463+0000 mon.a (mon.0) 563 : cluster [DBG] osdmap e502: 8 total, 8 up, 8 in 2024-09-13T00:43:18.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:18 smithi088 bash[68960]: audit 2024-09-13T00:43:17.129597+0000 mon.a (mon.0) 564 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-13T00:43:18.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:43:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:43:18] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.43.0" 2024-09-13T00:43:18.822 INFO:teuthology.orchestra.run.smithi077.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T00:43:18.822 INFO:teuthology.orchestra.run.smithi077.stdout:alertmanager.a smithi077 *:9093,9094 running (6m) 5s ago 13m 14.1M - 0.25.0 c8568f914cd2 49a4b5607300 2024-09-13T00:43:18.822 INFO:teuthology.orchestra.run.smithi077.stdout:grafana.a smithi088 *:3000 running (5m) 31s ago 12m 76.2M - 9.4.7 954c08fa6188 c369fdf58d31 2024-09-13T00:43:18.822 INFO:teuthology.orchestra.run.smithi077.stdout:mds.foo.smithi077.bigbbf smithi077 running (10m) 5s ago 10m 14.2M - 17.2.7-1627-gbbdadfa3 214985049bdb 30b25ab8f2dd 2024-09-13T00:43:18.823 INFO:teuthology.orchestra.run.smithi077.stdout:mds.foo.smithi088.krmowh smithi088 running (10m) 31s ago 10m 18.1M - 17.2.7-1627-gbbdadfa3 214985049bdb ebef6ef73335 2024-09-13T00:43:18.823 INFO:teuthology.orchestra.run.smithi077.stdout:mgr.x smithi088 *:8443,9283,8765 running (4m) 31s ago 21m 425M - 18.2.4-837-g7a853b17 f21ba1eea74e fdfc79d860d1 2024-09-13T00:43:18.823 INFO:teuthology.orchestra.run.smithi077.stdout:mgr.y smithi077 *:8443,9283,8765 running (5m) 5s ago 24m 483M - 18.2.4-837-g7a853b17 f21ba1eea74e d46e77b45675 2024-09-13T00:43:18.823 INFO:teuthology.orchestra.run.smithi077.stdout:mon.a smithi077 running (4m) 5s ago 24m 56.4M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 3edacf970db4 2024-09-13T00:43:18.823 INFO:teuthology.orchestra.run.smithi077.stdout:mon.b smithi088 running (2m) 31s ago 22m 34.2M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e dd7e71fd2ce3 2024-09-13T00:43:18.824 INFO:teuthology.orchestra.run.smithi077.stdout:mon.c smithi077 running (2m) 5s ago 22m 42.5M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 0a377e5bd304 2024-09-13T00:43:18.824 INFO:teuthology.orchestra.run.smithi077.stdout:node-exporter.a smithi077 *:9100 running (6m) 5s ago 13m 8351k - 1.5.0 0da6a335fe13 2aa0a10501f7 2024-09-13T00:43:18.824 INFO:teuthology.orchestra.run.smithi077.stdout:node-exporter.b smithi088 *:9100 running (5m) 31s ago 13m 8252k - 1.5.0 0da6a335fe13 54d4c3d89eab 2024-09-13T00:43:18.824 INFO:teuthology.orchestra.run.smithi077.stdout:osd.0 smithi077 running (7s) 5s ago 21m - 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 06ef1df834f4 2024-09-13T00:43:18.824 INFO:teuthology.orchestra.run.smithi077.stdout:osd.1 smithi077 running (20m) 5s ago 20m 121M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 8590b3b4d2e9 2024-09-13T00:43:18.824 INFO:teuthology.orchestra.run.smithi077.stdout:osd.2 smithi077 running (19m) 5s ago 19m 139M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb b2ecae227e58 2024-09-13T00:43:18.824 INFO:teuthology.orchestra.run.smithi077.stdout:osd.3 smithi077 running (18m) 5s ago 18m 122M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb d8c3c25f4ebd 2024-09-13T00:43:18.824 INFO:teuthology.orchestra.run.smithi077.stdout:osd.4 smithi088 running (17m) 31s ago 17m 115M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 2c9ea6919060 2024-09-13T00:43:18.824 INFO:teuthology.orchestra.run.smithi077.stdout:osd.5 smithi088 running (16m) 31s ago 16m 112M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb e89a6b298b28 2024-09-13T00:43:18.824 INFO:teuthology.orchestra.run.smithi077.stdout:osd.6 smithi088 running (15m) 31s ago 15m 135M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 599c9665873c 2024-09-13T00:43:18.824 INFO:teuthology.orchestra.run.smithi077.stdout:osd.7 smithi088 running (14m) 31s ago 14m 141M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 0ff0388ffa3c 2024-09-13T00:43:18.824 INFO:teuthology.orchestra.run.smithi077.stdout:prometheus.a smithi088 *:9095 running (4m) 31s ago 13m 54.4M - 2.43.0 a07b618ecd1d 5fb440283869 2024-09-13T00:43:19.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:19 smithi077 bash[80681]: cluster 2024-09-13T00:43:18.145514+0000 mon.a (mon.0) 565 : cluster [DBG] osdmap e503: 8 total, 8 up, 8 in 2024-09-13T00:43:19.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:19 smithi077 bash[83127]: cluster 2024-09-13T00:43:18.145514+0000 mon.a (mon.0) 565 : cluster [DBG] osdmap e503: 8 total, 8 up, 8 in 2024-09-13T00:43:19.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:19 smithi088 bash[68960]: cluster 2024-09-13T00:43:18.145514+0000 mon.a (mon.0) 565 : cluster [DBG] osdmap e503: 8 total, 8 up, 8 in 2024-09-13T00:43:20.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:20 smithi077 bash[83127]: cluster 2024-09-13T00:43:18.430936+0000 mgr.y (mgr.25304) 196 : cluster [DBG] pgmap v346: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 41/423 objects degraded (9.693%) 2024-09-13T00:43:20.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:20 smithi077 bash[83127]: audit 2024-09-13T00:43:18.800435+0000 mgr.y (mgr.25304) 197 : audit [DBG] from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:43:20.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:20 smithi077 bash[83127]: cluster 2024-09-13T00:43:19.149554+0000 mon.a (mon.0) 566 : cluster [DBG] osdmap e504: 8 total, 8 up, 8 in 2024-09-13T00:43:20.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:20 smithi077 bash[80681]: cluster 2024-09-13T00:43:18.430936+0000 mgr.y (mgr.25304) 196 : cluster [DBG] pgmap v346: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 41/423 objects degraded (9.693%) 2024-09-13T00:43:20.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:20 smithi077 bash[80681]: audit 2024-09-13T00:43:18.800435+0000 mgr.y (mgr.25304) 197 : audit [DBG] from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:43:20.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:20 smithi077 bash[80681]: cluster 2024-09-13T00:43:19.149554+0000 mon.a (mon.0) 566 : cluster [DBG] osdmap e504: 8 total, 8 up, 8 in 2024-09-13T00:43:20.552 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph orch upgrade pause' 2024-09-13T00:43:20.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:20 smithi088 bash[68960]: cluster 2024-09-13T00:43:18.430936+0000 mgr.y (mgr.25304) 196 : cluster [DBG] pgmap v346: 105 pgs: 19 active+undersized, 17 active+undersized+degraded, 69 active+clean; 584 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 41/423 objects degraded (9.693%) 2024-09-13T00:43:20.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:20 smithi088 bash[68960]: audit 2024-09-13T00:43:18.800435+0000 mgr.y (mgr.25304) 197 : audit [DBG] from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:43:20.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:20 smithi088 bash[68960]: cluster 2024-09-13T00:43:19.149554+0000 mon.a (mon.0) 566 : cluster [DBG] osdmap e504: 8 total, 8 up, 8 in 2024-09-13T00:43:21.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:21 smithi077 bash[80681]: cluster 2024-09-13T00:43:20.151476+0000 mon.a (mon.0) 567 : cluster [DBG] osdmap e505: 8 total, 8 up, 8 in 2024-09-13T00:43:21.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:21 smithi077 bash[80681]: cluster 2024-09-13T00:43:20.899728+0000 mon.a (mon.0) 568 : cluster [WRN] Health check update: Degraded data redundancy: 41/423 objects degraded (9.693%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:21.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:21 smithi077 bash[83127]: cluster 2024-09-13T00:43:20.151476+0000 mon.a (mon.0) 567 : cluster [DBG] osdmap e505: 8 total, 8 up, 8 in 2024-09-13T00:43:21.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:21 smithi077 bash[83127]: cluster 2024-09-13T00:43:20.899728+0000 mon.a (mon.0) 568 : cluster [WRN] Health check update: Degraded data redundancy: 41/423 objects degraded (9.693%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:21.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:21 smithi088 bash[68960]: cluster 2024-09-13T00:43:20.151476+0000 mon.a (mon.0) 567 : cluster [DBG] osdmap e505: 8 total, 8 up, 8 in 2024-09-13T00:43:21.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:21 smithi088 bash[68960]: cluster 2024-09-13T00:43:20.899728+0000 mon.a (mon.0) 568 : cluster [WRN] Health check update: Degraded data redundancy: 41/423 objects degraded (9.693%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:22.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:22 smithi077 bash[80681]: cluster 2024-09-13T00:43:20.431671+0000 mgr.y (mgr.25304) 198 : cluster [DBG] pgmap v349: 105 pgs: 9 active+undersized, 4 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 5.5 KiB/s wr, 104 op/s; 9/381 objects degraded (2.362%); 47 B/s, 2 keys/s, 3 objects/s recovering 2024-09-13T00:43:22.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:22 smithi077 bash[80681]: cluster 2024-09-13T00:43:21.177638+0000 mon.a (mon.0) 569 : cluster [DBG] osdmap e506: 8 total, 8 up, 8 in 2024-09-13T00:43:22.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:22 smithi077 bash[83127]: cluster 2024-09-13T00:43:20.431671+0000 mgr.y (mgr.25304) 198 : cluster [DBG] pgmap v349: 105 pgs: 9 active+undersized, 4 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 5.5 KiB/s wr, 104 op/s; 9/381 objects degraded (2.362%); 47 B/s, 2 keys/s, 3 objects/s recovering 2024-09-13T00:43:22.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:22 smithi077 bash[83127]: cluster 2024-09-13T00:43:21.177638+0000 mon.a (mon.0) 569 : cluster [DBG] osdmap e506: 8 total, 8 up, 8 in 2024-09-13T00:43:22.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:22 smithi088 bash[68960]: cluster 2024-09-13T00:43:20.431671+0000 mgr.y (mgr.25304) 198 : cluster [DBG] pgmap v349: 105 pgs: 9 active+undersized, 4 active+undersized+degraded, 92 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 5.5 KiB/s wr, 104 op/s; 9/381 objects degraded (2.362%); 47 B/s, 2 keys/s, 3 objects/s recovering 2024-09-13T00:43:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:22 smithi088 bash[68960]: cluster 2024-09-13T00:43:21.177638+0000 mon.a (mon.0) 569 : cluster [DBG] osdmap e506: 8 total, 8 up, 8 in 2024-09-13T00:43:22.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:43:22 smithi088 bash[61791]: ts=2024-09-13T00:43:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:43:23.459 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:23 smithi077 bash[80681]: cluster 2024-09-13T00:43:22.182293+0000 mon.a (mon.0) 570 : cluster [DBG] osdmap e507: 8 total, 8 up, 8 in 2024-09-13T00:43:23.459 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:23 smithi077 bash[83127]: cluster 2024-09-13T00:43:22.182293+0000 mon.a (mon.0) 570 : cluster [DBG] osdmap e507: 8 total, 8 up, 8 in 2024-09-13T00:43:23.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:23 smithi088 bash[68960]: cluster 2024-09-13T00:43:22.182293+0000 mon.a (mon.0) 570 : cluster [DBG] osdmap e507: 8 total, 8 up, 8 in 2024-09-13T00:43:24.465 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:24 smithi077 bash[80681]: cluster 2024-09-13T00:43:22.432416+0000 mgr.y (mgr.25304) 199 : cluster [DBG] pgmap v352: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 76 KiB/s rd, 6.5 KiB/s wr, 109 op/s; 60 B/s, 5 keys/s, 4 objects/s recovering 2024-09-13T00:43:24.465 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:24 smithi077 bash[80681]: cluster 2024-09-13T00:43:23.176036+0000 mon.a (mon.0) 571 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/381 objects degraded (2.362%), 4 pgs degraded) 2024-09-13T00:43:24.466 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:24 smithi077 bash[80681]: cluster 2024-09-13T00:43:23.192684+0000 mon.a (mon.0) 572 : cluster [DBG] osdmap e508: 8 total, 8 up, 8 in 2024-09-13T00:43:24.466 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:24 smithi077 bash[83127]: cluster 2024-09-13T00:43:22.432416+0000 mgr.y (mgr.25304) 199 : cluster [DBG] pgmap v352: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 76 KiB/s rd, 6.5 KiB/s wr, 109 op/s; 60 B/s, 5 keys/s, 4 objects/s recovering 2024-09-13T00:43:24.466 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:24 smithi077 bash[83127]: cluster 2024-09-13T00:43:23.176036+0000 mon.a (mon.0) 571 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/381 objects degraded (2.362%), 4 pgs degraded) 2024-09-13T00:43:24.466 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:24 smithi077 bash[83127]: cluster 2024-09-13T00:43:23.192684+0000 mon.a (mon.0) 572 : cluster [DBG] osdmap e508: 8 total, 8 up, 8 in 2024-09-13T00:43:24.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:24 smithi088 bash[68960]: cluster 2024-09-13T00:43:22.432416+0000 mgr.y (mgr.25304) 199 : cluster [DBG] pgmap v352: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 76 KiB/s rd, 6.5 KiB/s wr, 109 op/s; 60 B/s, 5 keys/s, 4 objects/s recovering 2024-09-13T00:43:24.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:24 smithi088 bash[68960]: cluster 2024-09-13T00:43:23.176036+0000 mon.a (mon.0) 571 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9/381 objects degraded (2.362%), 4 pgs degraded) 2024-09-13T00:43:24.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:24 smithi088 bash[68960]: cluster 2024-09-13T00:43:23.192684+0000 mon.a (mon.0) 572 : cluster [DBG] osdmap e508: 8 total, 8 up, 8 in 2024-09-13T00:43:25.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:25 smithi077 bash[80681]: cluster 2024-09-13T00:43:24.193274+0000 mon.a (mon.0) 573 : cluster [DBG] osdmap e509: 8 total, 8 up, 8 in 2024-09-13T00:43:25.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:25 smithi077 bash[83127]: cluster 2024-09-13T00:43:24.193274+0000 mon.a (mon.0) 573 : cluster [DBG] osdmap e509: 8 total, 8 up, 8 in 2024-09-13T00:43:25.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:25 smithi088 bash[68960]: cluster 2024-09-13T00:43:24.193274+0000 mon.a (mon.0) 573 : cluster [DBG] osdmap e509: 8 total, 8 up, 8 in 2024-09-13T00:43:25.812 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:43:26.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[83127]: cluster 2024-09-13T00:43:24.433188+0000 mgr.y (mgr.25304) 200 : cluster [DBG] pgmap v355: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 79 KiB/s rd, 7.5 KiB/s wr, 114 op/s; 12 B/s, 3 keys/s, 1 objects/s recovering 2024-09-13T00:43:26.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[83127]: cluster 2024-09-13T00:43:25.213802+0000 mon.a (mon.0) 574 : cluster [DBG] osdmap e510: 8 total, 8 up, 8 in 2024-09-13T00:43:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[83127]: audit 2024-09-13T00:43:25.356968+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[83127]: audit 2024-09-13T00:43:25.367207+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[83127]: audit 2024-09-13T00:43:25.369048+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:43:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[83127]: audit 2024-09-13T00:43:25.370683+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:43:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[83127]: audit 2024-09-13T00:43:25.381049+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[83127]: audit 2024-09-13T00:43:25.502453+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:43:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[83127]: audit 2024-09-13T00:43:25.507354+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:43:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[83127]: audit 2024-09-13T00:43:25.510100+0000 mon.a (mon.0) 582 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:43:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[83127]: audit 2024-09-13T00:43:25.512304+0000 mon.a (mon.0) 583 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:43:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[83127]: audit 2024-09-13T00:43:25.514812+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T00:43:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[83127]: cluster 2024-09-13T00:43:25.912377+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e511: 8 total, 8 up, 8 in 2024-09-13T00:43:26.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[80681]: cluster 2024-09-13T00:43:24.433188+0000 mgr.y (mgr.25304) 200 : cluster [DBG] pgmap v355: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 79 KiB/s rd, 7.5 KiB/s wr, 114 op/s; 12 B/s, 3 keys/s, 1 objects/s recovering 2024-09-13T00:43:26.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[80681]: cluster 2024-09-13T00:43:25.213802+0000 mon.a (mon.0) 574 : cluster [DBG] osdmap e510: 8 total, 8 up, 8 in 2024-09-13T00:43:26.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[80681]: audit 2024-09-13T00:43:25.356968+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:26.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[80681]: audit 2024-09-13T00:43:25.367207+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:26.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[80681]: audit 2024-09-13T00:43:25.369048+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:43:26.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[80681]: audit 2024-09-13T00:43:25.370683+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:43:26.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[80681]: audit 2024-09-13T00:43:25.381049+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:26.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[80681]: audit 2024-09-13T00:43:25.502453+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:43:26.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[80681]: audit 2024-09-13T00:43:25.507354+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:43:26.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[80681]: audit 2024-09-13T00:43:25.510100+0000 mon.a (mon.0) 582 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:43:26.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[80681]: audit 2024-09-13T00:43:25.512304+0000 mon.a (mon.0) 583 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:43:26.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[80681]: audit 2024-09-13T00:43:25.514812+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T00:43:26.531 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:26 smithi077 bash[80681]: cluster 2024-09-13T00:43:25.912377+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e511: 8 total, 8 up, 8 in 2024-09-13T00:43:26.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:26 smithi088 bash[68960]: cluster 2024-09-13T00:43:24.433188+0000 mgr.y (mgr.25304) 200 : cluster [DBG] pgmap v355: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 79 KiB/s rd, 7.5 KiB/s wr, 114 op/s; 12 B/s, 3 keys/s, 1 objects/s recovering 2024-09-13T00:43:26.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:26 smithi088 bash[68960]: cluster 2024-09-13T00:43:25.213802+0000 mon.a (mon.0) 574 : cluster [DBG] osdmap e510: 8 total, 8 up, 8 in 2024-09-13T00:43:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:26 smithi088 bash[68960]: audit 2024-09-13T00:43:25.356968+0000 mon.a (mon.0) 575 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:26 smithi088 bash[68960]: audit 2024-09-13T00:43:25.367207+0000 mon.a (mon.0) 576 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:26 smithi088 bash[68960]: audit 2024-09-13T00:43:25.369048+0000 mon.a (mon.0) 577 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:43:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:26 smithi088 bash[68960]: audit 2024-09-13T00:43:25.370683+0000 mon.a (mon.0) 578 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:43:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:26 smithi088 bash[68960]: audit 2024-09-13T00:43:25.381049+0000 mon.a (mon.0) 579 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:26 smithi088 bash[68960]: audit 2024-09-13T00:43:25.502453+0000 mon.a (mon.0) 580 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:43:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:26 smithi088 bash[68960]: audit 2024-09-13T00:43:25.507354+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:43:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:26 smithi088 bash[68960]: audit 2024-09-13T00:43:25.510100+0000 mon.a (mon.0) 582 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:43:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:26 smithi088 bash[68960]: audit 2024-09-13T00:43:25.512304+0000 mon.a (mon.0) 583 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:43:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:26 smithi088 bash[68960]: audit 2024-09-13T00:43:25.514812+0000 mon.a (mon.0) 584 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T00:43:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:26 smithi088 bash[68960]: cluster 2024-09-13T00:43:25.912377+0000 mon.a (mon.0) 585 : cluster [DBG] osdmap e511: 8 total, 8 up, 8 in 2024-09-13T00:43:27.049 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestMirroring.AioSnapshotCreate (32795 ms) 2024-09-13T00:43:27.556 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:27 smithi077 bash[80681]: audit 2024-09-13T00:43:25.515244+0000 mgr.y (mgr.25304) 201 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T00:43:27.556 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:27 smithi077 bash[80681]: cephadm 2024-09-13T00:43:25.517210+0000 mgr.y (mgr.25304) 202 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-09-13T00:43:27.556 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:27 smithi077 bash[80681]: cluster 2024-09-13T00:43:26.914100+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e512: 8 total, 8 up, 8 in 2024-09-13T00:43:27.557 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:27 smithi077 bash[80681]: audit 2024-09-13T00:43:27.030981+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/93acc6e5-74af-41ba-bdc3-fa61df5fac2b"}]: dispatch 2024-09-13T00:43:27.557 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:27 smithi077 bash[80681]: audit 2024-09-13T00:43:27.031663+0000 mon.a (mon.0) 587 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/93acc6e5-74af-41ba-bdc3-fa61df5fac2b"}]: dispatch 2024-09-13T00:43:27.557 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:27 smithi077 bash[80681]: audit 2024-09-13T00:43:27.038709+0000 mon.a (mon.0) 588 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/93acc6e5-74af-41ba-bdc3-fa61df5fac2b"}]': finished 2024-09-13T00:43:27.557 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:27 smithi077 bash[83127]: audit 2024-09-13T00:43:25.515244+0000 mgr.y (mgr.25304) 201 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T00:43:27.557 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:27 smithi077 bash[83127]: cephadm 2024-09-13T00:43:25.517210+0000 mgr.y (mgr.25304) 202 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-09-13T00:43:27.557 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:27 smithi077 bash[83127]: cluster 2024-09-13T00:43:26.914100+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e512: 8 total, 8 up, 8 in 2024-09-13T00:43:27.557 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:27 smithi077 bash[83127]: audit 2024-09-13T00:43:27.030981+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/93acc6e5-74af-41ba-bdc3-fa61df5fac2b"}]: dispatch 2024-09-13T00:43:27.557 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:27 smithi077 bash[83127]: audit 2024-09-13T00:43:27.031663+0000 mon.a (mon.0) 587 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/93acc6e5-74af-41ba-bdc3-fa61df5fac2b"}]: dispatch 2024-09-13T00:43:27.557 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:27 smithi077 bash[83127]: audit 2024-09-13T00:43:27.038709+0000 mon.a (mon.0) 588 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/93acc6e5-74af-41ba-bdc3-fa61df5fac2b"}]': finished 2024-09-13T00:43:27.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:27 smithi088 bash[68960]: audit 2024-09-13T00:43:25.515244+0000 mgr.y (mgr.25304) 201 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-13T00:43:27.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:27 smithi088 bash[68960]: cephadm 2024-09-13T00:43:25.517210+0000 mgr.y (mgr.25304) 202 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-09-13T00:43:27.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:27 smithi088 bash[68960]: cluster 2024-09-13T00:43:26.914100+0000 mon.a (mon.0) 586 : cluster [DBG] osdmap e512: 8 total, 8 up, 8 in 2024-09-13T00:43:27.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:27 smithi088 bash[68960]: audit 2024-09-13T00:43:27.030981+0000 mon.c (mon.2) 17 : audit [INF] from='client.? 172.21.15.77:0/1060205356' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/93acc6e5-74af-41ba-bdc3-fa61df5fac2b"}]: dispatch 2024-09-13T00:43:27.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:27 smithi088 bash[68960]: audit 2024-09-13T00:43:27.031663+0000 mon.a (mon.0) 587 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/93acc6e5-74af-41ba-bdc3-fa61df5fac2b"}]: dispatch 2024-09-13T00:43:27.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:27 smithi088 bash[68960]: audit 2024-09-13T00:43:27.038709+0000 mon.a (mon.0) 588 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/15/93acc6e5-74af-41ba-bdc3-fa61df5fac2b"}]': finished 2024-09-13T00:43:28.051 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 51 tests from TestMirroring (95129 ms total) 2024-09-13T00:43:28.052 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:43:28.052 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 34 tests from TestInternal 2024-09-13T00:43:28.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:43:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:43:28] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.43.0" 2024-09-13T00:43:28.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:28 smithi077 bash[80681]: cluster 2024-09-13T00:43:26.433937+0000 mgr.y (mgr.25304) 203 : cluster [DBG] pgmap v358: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 8.0 KiB/s wr, 119 op/s 2024-09-13T00:43:28.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:28 smithi077 bash[80681]: audit 2024-09-13T00:43:27.666213+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:28.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:28 smithi077 bash[80681]: cluster 2024-09-13T00:43:28.042846+0000 mon.a (mon.0) 590 : cluster [DBG] osdmap e513: 8 total, 8 up, 8 in 2024-09-13T00:43:28.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:28 smithi077 bash[83127]: cluster 2024-09-13T00:43:26.433937+0000 mgr.y (mgr.25304) 203 : cluster [DBG] pgmap v358: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 8.0 KiB/s wr, 119 op/s 2024-09-13T00:43:28.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:28 smithi077 bash[83127]: audit 2024-09-13T00:43:27.666213+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:28.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:28 smithi077 bash[83127]: cluster 2024-09-13T00:43:28.042846+0000 mon.a (mon.0) 590 : cluster [DBG] osdmap e513: 8 total, 8 up, 8 in 2024-09-13T00:43:28.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:28 smithi088 bash[68960]: cluster 2024-09-13T00:43:26.433937+0000 mgr.y (mgr.25304) 203 : cluster [DBG] pgmap v358: 105 pgs: 105 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 81 KiB/s rd, 8.0 KiB/s wr, 119 op/s 2024-09-13T00:43:28.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:28 smithi088 bash[68960]: audit 2024-09-13T00:43:27.666213+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:28.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:28 smithi088 bash[68960]: cluster 2024-09-13T00:43:28.042846+0000 mon.a (mon.0) 590 : cluster [DBG] osdmap e513: 8 total, 8 up, 8 in 2024-09-13T00:43:29.840 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:29 smithi077 bash[80681]: cluster 2024-09-13T00:43:28.434471+0000 mgr.y (mgr.25304) 204 : cluster [DBG] pgmap v361: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:43:29.841 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:29 smithi077 bash[80681]: audit 2024-09-13T00:43:28.548129+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:29.841 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:29 smithi077 bash[80681]: audit 2024-09-13T00:43:28.549209+0000 mon.a (mon.0) 592 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:29.841 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:29 smithi077 bash[80681]: cluster 2024-09-13T00:43:29.052317+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e514: 8 total, 8 up, 8 in 2024-09-13T00:43:29.841 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:29 smithi077 bash[80681]: audit 2024-09-13T00:43:29.054230+0000 mon.c (mon.2) 18 : audit [INF] from='client.? 172.21.15.77:0/1049986590' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:43:29.841 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:29 smithi077 bash[80681]: audit 2024-09-13T00:43:29.054717+0000 mon.a (mon.0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:43:29.841 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:29 smithi077 bash[83127]: cluster 2024-09-13T00:43:28.434471+0000 mgr.y (mgr.25304) 204 : cluster [DBG] pgmap v361: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:43:29.841 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:29 smithi077 bash[83127]: audit 2024-09-13T00:43:28.548129+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:29.841 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:29 smithi077 bash[83127]: audit 2024-09-13T00:43:28.549209+0000 mon.a (mon.0) 592 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:29.841 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:29 smithi077 bash[83127]: cluster 2024-09-13T00:43:29.052317+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e514: 8 total, 8 up, 8 in 2024-09-13T00:43:29.841 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:29 smithi077 bash[83127]: audit 2024-09-13T00:43:29.054230+0000 mon.c (mon.2) 18 : audit [INF] from='client.? 172.21.15.77:0/1049986590' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:43:29.842 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:29 smithi077 bash[83127]: audit 2024-09-13T00:43:29.054717+0000 mon.a (mon.0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:43:29.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:29 smithi088 bash[68960]: cluster 2024-09-13T00:43:28.434471+0000 mgr.y (mgr.25304) 204 : cluster [DBG] pgmap v361: 73 pgs: 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:43:29.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:29 smithi088 bash[68960]: audit 2024-09-13T00:43:28.548129+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:29.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:29 smithi088 bash[68960]: audit 2024-09-13T00:43:28.549209+0000 mon.a (mon.0) 592 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:29.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:29 smithi088 bash[68960]: cluster 2024-09-13T00:43:29.052317+0000 mon.a (mon.0) 593 : cluster [DBG] osdmap e514: 8 total, 8 up, 8 in 2024-09-13T00:43:29.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:29 smithi088 bash[68960]: audit 2024-09-13T00:43:29.054230+0000 mon.c (mon.2) 18 : audit [INF] from='client.? 172.21.15.77:0/1049986590' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:43:29.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:29 smithi088 bash[68960]: audit 2024-09-13T00:43:29.054717+0000 mon.a (mon.0) 594 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:43:30.054 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.OpenByID 2024-09-13T00:43:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:31 smithi088 bash[68960]: audit 2024-09-13T00:43:30.054958+0000 mon.a (mon.0) 595 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:43:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:31 smithi088 bash[68960]: cluster 2024-09-13T00:43:30.056366+0000 mon.a (mon.0) 596 : cluster [DBG] osdmap e515: 8 total, 8 up, 8 in 2024-09-13T00:43:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:31 smithi088 bash[68960]: audit 2024-09-13T00:43:30.526006+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:31 smithi088 bash[68960]: audit 2024-09-13T00:43:30.527163+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T00:43:31.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:31 smithi088 bash[68960]: audit 2024-09-13T00:43:30.527909+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:43:31.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:31 smithi077 bash[80681]: audit 2024-09-13T00:43:30.054958+0000 mon.a (mon.0) 595 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:43:31.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:31 smithi077 bash[80681]: cluster 2024-09-13T00:43:30.056366+0000 mon.a (mon.0) 596 : cluster [DBG] osdmap e515: 8 total, 8 up, 8 in 2024-09-13T00:43:31.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:31 smithi077 bash[80681]: audit 2024-09-13T00:43:30.526006+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:31.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:31 smithi077 bash[80681]: audit 2024-09-13T00:43:30.527163+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T00:43:31.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:31 smithi077 bash[80681]: audit 2024-09-13T00:43:30.527909+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:43:31.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:31 smithi077 bash[83127]: audit 2024-09-13T00:43:30.054958+0000 mon.a (mon.0) 595 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:43:31.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:31 smithi077 bash[83127]: cluster 2024-09-13T00:43:30.056366+0000 mon.a (mon.0) 596 : cluster [DBG] osdmap e515: 8 total, 8 up, 8 in 2024-09-13T00:43:31.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:31 smithi077 bash[83127]: audit 2024-09-13T00:43:30.526006+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:31.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:31 smithi077 bash[83127]: audit 2024-09-13T00:43:30.527163+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-13T00:43:31.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:31 smithi077 bash[83127]: audit 2024-09-13T00:43:30.527909+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:43:32.138 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.OpenByID (2084 ms) 2024-09-13T00:43:32.138 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.OpenSnapDNE 2024-09-13T00:43:32.192 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.OpenSnapDNE (54 ms) 2024-09-13T00:43:32.193 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.IsExclusiveLockOwner 2024-09-13T00:43:32.238 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.IsExclusiveLockOwner (47 ms) 2024-09-13T00:43:32.239 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.ResizeLocksImage 2024-09-13T00:43:32.303 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.ResizeLocksImage (64 ms) 2024-09-13T00:43:32.303 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.ResizeFailsToLockImage 2024-09-13T00:43:32.341 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.ResizeFailsToLockImage (38 ms) 2024-09-13T00:43:32.341 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.SnapCreateLocksImage 2024-09-13T00:43:32.351 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:43:32 smithi088 bash[61791]: ts=2024-09-13T00:43:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:43:32.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:32 smithi088 bash[68960]: cluster 2024-09-13T00:43:30.434959+0000 mgr.y (mgr.25304) 205 : cluster [DBG] pgmap v364: 105 pgs: 25 creating+peering, 7 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:43:32.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:32 smithi088 bash[68960]: cephadm 2024-09-13T00:43:30.520738+0000 mgr.y (mgr.25304) 206 : cephadm [INF] Upgrade: Updating osd.1 2024-09-13T00:43:32.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:32 smithi088 bash[68960]: cephadm 2024-09-13T00:43:30.530343+0000 mgr.y (mgr.25304) 207 : cephadm [INF] Deploying daemon osd.1 on smithi077 2024-09-13T00:43:32.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:32 smithi088 bash[68960]: cluster 2024-09-13T00:43:31.070464+0000 mon.a (mon.0) 600 : cluster [DBG] osdmap e516: 8 total, 8 up, 8 in 2024-09-13T00:43:32.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:32 smithi077 bash[80681]: cluster 2024-09-13T00:43:30.434959+0000 mgr.y (mgr.25304) 205 : cluster [DBG] pgmap v364: 105 pgs: 25 creating+peering, 7 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:43:32.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:32 smithi077 bash[80681]: cephadm 2024-09-13T00:43:30.520738+0000 mgr.y (mgr.25304) 206 : cephadm [INF] Upgrade: Updating osd.1 2024-09-13T00:43:32.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:32 smithi077 bash[80681]: cephadm 2024-09-13T00:43:30.530343+0000 mgr.y (mgr.25304) 207 : cephadm [INF] Deploying daemon osd.1 on smithi077 2024-09-13T00:43:32.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:32 smithi077 bash[80681]: cluster 2024-09-13T00:43:31.070464+0000 mon.a (mon.0) 600 : cluster [DBG] osdmap e516: 8 total, 8 up, 8 in 2024-09-13T00:43:32.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:32 smithi077 bash[83127]: cluster 2024-09-13T00:43:30.434959+0000 mgr.y (mgr.25304) 205 : cluster [DBG] pgmap v364: 105 pgs: 25 creating+peering, 7 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:43:32.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:32 smithi077 bash[83127]: cephadm 2024-09-13T00:43:30.520738+0000 mgr.y (mgr.25304) 206 : cephadm [INF] Upgrade: Updating osd.1 2024-09-13T00:43:32.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:32 smithi077 bash[83127]: cephadm 2024-09-13T00:43:30.530343+0000 mgr.y (mgr.25304) 207 : cephadm [INF] Deploying daemon osd.1 on smithi077 2024-09-13T00:43:32.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:32 smithi077 bash[83127]: cluster 2024-09-13T00:43:31.070464+0000 mon.a (mon.0) 600 : cluster [DBG] osdmap e516: 8 total, 8 up, 8 in 2024-09-13T00:43:32.968 INFO:teuthology.orchestra.run.smithi077.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:43:33.238 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:33 smithi077 bash[80681]: cluster 2024-09-13T00:43:32.068508+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e517: 8 total, 8 up, 8 in 2024-09-13T00:43:33.238 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:33 smithi077 bash[80681]: audit 2024-09-13T00:43:32.970907+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:33.238 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:33 smithi077 bash[83127]: cluster 2024-09-13T00:43:32.068508+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e517: 8 total, 8 up, 8 in 2024-09-13T00:43:33.239 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:33 smithi077 bash[83127]: audit 2024-09-13T00:43:32.970907+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:33.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:33 smithi088 bash[68960]: cluster 2024-09-13T00:43:32.068508+0000 mon.a (mon.0) 601 : cluster [DBG] osdmap e517: 8 total, 8 up, 8 in 2024-09-13T00:43:33.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:33 smithi088 bash[68960]: audit 2024-09-13T00:43:32.970907+0000 mon.a (mon.0) 602 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:34.122 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.SnapCreateLocksImage (1781 ms) 2024-09-13T00:43:34.122 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.SnapCreateFailsToLockImage 2024-09-13T00:43:34.180 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.SnapCreateFailsToLockImage (58 ms) 2024-09-13T00:43:34.180 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.SnapRollbackLocksImage 2024-09-13T00:43:34.243 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'sleep 60' 2024-09-13T00:43:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:34 smithi088 bash[68960]: cluster 2024-09-13T00:43:32.435400+0000 mgr.y (mgr.25304) 208 : cluster [DBG] pgmap v367: 105 pgs: 25 creating+peering, 80 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:43:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:34 smithi088 bash[68960]: audit 2024-09-13T00:43:32.965771+0000 mgr.y (mgr.25304) 209 : audit [DBG] from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:43:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:34 smithi088 bash[68960]: cephadm 2024-09-13T00:43:32.966161+0000 mgr.y (mgr.25304) 210 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:43:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:34 smithi088 bash[68960]: cluster 2024-09-13T00:43:33.091500+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e518: 8 total, 8 up, 8 in 2024-09-13T00:43:34.424 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:34 smithi077 bash[83127]: cluster 2024-09-13T00:43:32.435400+0000 mgr.y (mgr.25304) 208 : cluster [DBG] pgmap v367: 105 pgs: 25 creating+peering, 80 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:43:34.424 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:34 smithi077 bash[83127]: audit 2024-09-13T00:43:32.965771+0000 mgr.y (mgr.25304) 209 : audit [DBG] from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:43:34.424 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:34 smithi077 bash[83127]: cephadm 2024-09-13T00:43:32.966161+0000 mgr.y (mgr.25304) 210 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:43:34.424 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:34 smithi077 bash[83127]: cluster 2024-09-13T00:43:33.091500+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e518: 8 total, 8 up, 8 in 2024-09-13T00:43:34.425 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:34 smithi077 bash[80681]: cluster 2024-09-13T00:43:32.435400+0000 mgr.y (mgr.25304) 208 : cluster [DBG] pgmap v367: 105 pgs: 25 creating+peering, 80 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:43:34.425 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:34 smithi077 bash[80681]: audit 2024-09-13T00:43:32.965771+0000 mgr.y (mgr.25304) 209 : audit [DBG] from='client.44174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:43:34.425 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:34 smithi077 bash[80681]: cephadm 2024-09-13T00:43:32.966161+0000 mgr.y (mgr.25304) 210 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:43:34.425 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:34 smithi077 bash[80681]: cluster 2024-09-13T00:43:33.091500+0000 mon.a (mon.0) 603 : cluster [DBG] osdmap e518: 8 total, 8 up, 8 in 2024-09-13T00:43:35.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:35 smithi077 bash[80681]: cluster 2024-09-13T00:43:34.104073+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e519: 8 total, 8 up, 8 in 2024-09-13T00:43:35.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:35 smithi077 bash[83127]: cluster 2024-09-13T00:43:34.104073+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e519: 8 total, 8 up, 8 in 2024-09-13T00:43:35.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:35 smithi088 bash[68960]: cluster 2024-09-13T00:43:34.104073+0000 mon.a (mon.0) 604 : cluster [DBG] osdmap e519: 8 total, 8 up, 8 in 2024-09-13T00:43:36.159 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.SnapRollbackLocksImage (1980 ms) 2024-09-13T00:43:36.159 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.SnapRollbackFailsToLockImage 2024-09-13T00:43:36.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:36 smithi077 bash[80681]: cluster 2024-09-13T00:43:34.436015+0000 mgr.y (mgr.25304) 211 : cluster [DBG] pgmap v370: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 8.0 KiB/s wr, 85 op/s 2024-09-13T00:43:36.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:36 smithi077 bash[80681]: cluster 2024-09-13T00:43:35.114912+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2024-09-13T00:43:36.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:36 smithi077 bash[83127]: cluster 2024-09-13T00:43:34.436015+0000 mgr.y (mgr.25304) 211 : cluster [DBG] pgmap v370: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 8.0 KiB/s wr, 85 op/s 2024-09-13T00:43:36.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:36 smithi077 bash[83127]: cluster 2024-09-13T00:43:35.114912+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2024-09-13T00:43:36.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:36 smithi088 bash[68960]: cluster 2024-09-13T00:43:34.436015+0000 mgr.y (mgr.25304) 211 : cluster [DBG] pgmap v370: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 8.0 KiB/s wr, 85 op/s 2024-09-13T00:43:36.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:36 smithi088 bash[68960]: cluster 2024-09-13T00:43:35.114912+0000 mon.a (mon.0) 605 : cluster [DBG] osdmap e520: 8 total, 8 up, 8 in 2024-09-13T00:43:37.265 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:43:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:37.265 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:43:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:37.266 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:43:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:37.266 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:37.266 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:43:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:37.266 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:37.267 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:37 smithi077 bash[83127]: cluster 2024-09-13T00:43:36.115824+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2024-09-13T00:43:37.267 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:37.267 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:37 smithi077 bash[80681]: cluster 2024-09-13T00:43:36.115824+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2024-09-13T00:43:37.267 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:37.267 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:43:37 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:37.528 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:37 smithi077 systemd[1]: Stopping Ceph osd.1 for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:43:37.528 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:37 smithi077 bash[40438]: debug 2024-09-13T00:43:37.366+0000 7fd062d47640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:43:37.528 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:37 smithi077 bash[40438]: debug 2024-09-13T00:43:37.366+0000 7fd062d47640 -1 osd.1 522 *** Got signal Terminated *** 2024-09-13T00:43:37.529 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:37 smithi077 bash[40438]: debug 2024-09-13T00:43:37.366+0000 7fd062d47640 -1 osd.1 522 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T00:43:37.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:37 smithi088 bash[68960]: cluster 2024-09-13T00:43:36.115824+0000 mon.a (mon.0) 606 : cluster [DBG] osdmap e521: 8 total, 8 up, 8 in 2024-09-13T00:43:38.194 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.SnapRollbackFailsToLockImage (2034 ms) 2024-09-13T00:43:38.195 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.SnapSetReleasesLock 2024-09-13T00:43:38.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:38 smithi077 bash[80681]: debug 2024-09-13T00:43:38.126+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:43:38.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:38 smithi077 bash[80681]: cluster 2024-09-13T00:43:36.436880+0000 mgr.y (mgr.25304) 212 : cluster [DBG] pgmap v373: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 11 KiB/s wr, 122 op/s 2024-09-13T00:43:38.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:38 smithi077 bash[80681]: cluster 2024-09-13T00:43:37.135344+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2024-09-13T00:43:38.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:38 smithi077 bash[80681]: cluster 2024-09-13T00:43:37.374371+0000 mon.a (mon.0) 608 : cluster [INF] osd.1 marked itself down and dead 2024-09-13T00:43:38.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:38 smithi077 bash[83127]: cluster 2024-09-13T00:43:36.436880+0000 mgr.y (mgr.25304) 212 : cluster [DBG] pgmap v373: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 11 KiB/s wr, 122 op/s 2024-09-13T00:43:38.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:38 smithi077 bash[83127]: cluster 2024-09-13T00:43:37.135344+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2024-09-13T00:43:38.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:38 smithi077 bash[83127]: cluster 2024-09-13T00:43:37.374371+0000 mon.a (mon.0) 608 : cluster [INF] osd.1 marked itself down and dead 2024-09-13T00:43:38.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:38 smithi088 bash[68960]: cluster 2024-09-13T00:43:36.436880+0000 mgr.y (mgr.25304) 212 : cluster [DBG] pgmap v373: 105 pgs: 105 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 82 KiB/s rd, 11 KiB/s wr, 122 op/s 2024-09-13T00:43:38.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:38 smithi088 bash[68960]: cluster 2024-09-13T00:43:37.135344+0000 mon.a (mon.0) 607 : cluster [DBG] osdmap e522: 8 total, 8 up, 8 in 2024-09-13T00:43:38.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:38 smithi088 bash[68960]: cluster 2024-09-13T00:43:37.374371+0000 mon.a (mon.0) 608 : cluster [INF] osd.1 marked itself down and dead 2024-09-13T00:43:38.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:43:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:43:38] "GET /metrics HTTP/1.1" 200 34141 "" "Prometheus/2.43.0" 2024-09-13T00:43:39.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:39 smithi077 bash[83127]: cluster 2024-09-13T00:43:38.132122+0000 mon.a (mon.0) 609 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T00:43:39.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:39 smithi077 bash[83127]: cluster 2024-09-13T00:43:38.132169+0000 mon.a (mon.0) 610 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:43:39.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:39 smithi077 bash[83127]: cluster 2024-09-13T00:43:38.151189+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e523: 8 total, 7 up, 8 in 2024-09-13T00:43:39.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:39 smithi077 bash[80681]: cluster 2024-09-13T00:43:38.132122+0000 mon.a (mon.0) 609 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T00:43:39.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:39 smithi077 bash[80681]: cluster 2024-09-13T00:43:38.132169+0000 mon.a (mon.0) 610 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:43:39.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:39 smithi077 bash[80681]: cluster 2024-09-13T00:43:38.151189+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e523: 8 total, 7 up, 8 in 2024-09-13T00:43:39.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:39 smithi088 bash[68960]: cluster 2024-09-13T00:43:38.132122+0000 mon.a (mon.0) 609 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T00:43:39.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:39 smithi088 bash[68960]: cluster 2024-09-13T00:43:38.132169+0000 mon.a (mon.0) 610 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:43:39.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:39 smithi088 bash[68960]: cluster 2024-09-13T00:43:38.151189+0000 mon.a (mon.0) 611 : cluster [DBG] osdmap e523: 8 total, 7 up, 8 in 2024-09-13T00:43:39.717 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:39 smithi077 bash[94114]: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-1 2024-09-13T00:43:40.028 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:39 smithi077 bash[94181]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-1 2024-09-13T00:43:40.232 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:43:40.512 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:40 smithi077 bash[80681]: cluster 2024-09-13T00:43:38.437689+0000 mgr.y (mgr.25304) 213 : cluster [DBG] pgmap v376: 105 pgs: 13 stale+active+clean, 92 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 KiB/s wr, 37 op/s 2024-09-13T00:43:40.512 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:40 smithi077 bash[80681]: cluster 2024-09-13T00:43:39.157375+0000 mon.a (mon.0) 612 : cluster [DBG] osdmap e524: 8 total, 7 up, 8 in 2024-09-13T00:43:40.512 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:40 smithi077 bash[83127]: cluster 2024-09-13T00:43:38.437689+0000 mgr.y (mgr.25304) 213 : cluster [DBG] pgmap v376: 105 pgs: 13 stale+active+clean, 92 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 KiB/s wr, 37 op/s 2024-09-13T00:43:40.512 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:40 smithi077 bash[83127]: cluster 2024-09-13T00:43:39.157375+0000 mon.a (mon.0) 612 : cluster [DBG] osdmap e524: 8 total, 7 up, 8 in 2024-09-13T00:43:40.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:40 smithi088 bash[68960]: cluster 2024-09-13T00:43:38.437689+0000 mgr.y (mgr.25304) 213 : cluster [DBG] pgmap v376: 105 pgs: 13 stale+active+clean, 92 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 KiB/s wr, 37 op/s 2024-09-13T00:43:40.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:40 smithi088 bash[68960]: cluster 2024-09-13T00:43:39.157375+0000 mon.a (mon.0) 612 : cluster [DBG] osdmap e524: 8 total, 7 up, 8 in 2024-09-13T00:43:40.940 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.SnapSetReleasesLock (2745 ms) 2024-09-13T00:43:40.940 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.FlattenLocksImage 2024-09-13T00:43:41.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:41 smithi077 bash[83127]: cluster 2024-09-13T00:43:40.160035+0000 mon.a (mon.0) 613 : cluster [DBG] osdmap e525: 8 total, 7 up, 8 in 2024-09-13T00:43:41.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:41 smithi077 bash[83127]: cluster 2024-09-13T00:43:40.915837+0000 mon.a (mon.0) 614 : cluster [DBG] osdmap e526: 8 total, 7 up, 8 in 2024-09-13T00:43:41.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:41 smithi077 bash[80681]: cluster 2024-09-13T00:43:40.160035+0000 mon.a (mon.0) 613 : cluster [DBG] osdmap e525: 8 total, 7 up, 8 in 2024-09-13T00:43:41.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:41 smithi077 bash[80681]: cluster 2024-09-13T00:43:40.915837+0000 mon.a (mon.0) 614 : cluster [DBG] osdmap e526: 8 total, 7 up, 8 in 2024-09-13T00:43:41.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:41 smithi088 bash[68960]: cluster 2024-09-13T00:43:40.160035+0000 mon.a (mon.0) 613 : cluster [DBG] osdmap e525: 8 total, 7 up, 8 in 2024-09-13T00:43:41.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:41 smithi088 bash[68960]: cluster 2024-09-13T00:43:40.915837+0000 mon.a (mon.0) 614 : cluster [DBG] osdmap e526: 8 total, 7 up, 8 in 2024-09-13T00:43:42.427 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:42 smithi077 bash[80681]: cluster 2024-09-13T00:43:40.438618+0000 mgr.y (mgr.25304) 214 : cluster [DBG] pgmap v379: 105 pgs: 32 active+undersized, 1 stale+active+clean, 11 active+undersized+degraded, 61 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s rd, 5.0 KiB/s wr, 82 op/s; 13/168 objects degraded (7.738%) 2024-09-13T00:43:42.427 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:42 smithi077 bash[80681]: cluster 2024-09-13T00:43:41.161812+0000 mon.a (mon.0) 615 : cluster [WRN] Health check failed: Degraded data redundancy: 13/168 objects degraded (7.738%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:42.428 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:42 smithi077 bash[80681]: cluster 2024-09-13T00:43:41.916195+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e527: 8 total, 7 up, 8 in 2024-09-13T00:43:42.428 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:42 smithi077 bash[83127]: cluster 2024-09-13T00:43:40.438618+0000 mgr.y (mgr.25304) 214 : cluster [DBG] pgmap v379: 105 pgs: 32 active+undersized, 1 stale+active+clean, 11 active+undersized+degraded, 61 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s rd, 5.0 KiB/s wr, 82 op/s; 13/168 objects degraded (7.738%) 2024-09-13T00:43:42.428 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:42 smithi077 bash[83127]: cluster 2024-09-13T00:43:41.161812+0000 mon.a (mon.0) 615 : cluster [WRN] Health check failed: Degraded data redundancy: 13/168 objects degraded (7.738%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:42.428 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:42 smithi077 bash[83127]: cluster 2024-09-13T00:43:41.916195+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e527: 8 total, 7 up, 8 in 2024-09-13T00:43:42.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:43:42 smithi088 bash[61791]: ts=2024-09-13T00:43:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:43:42.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:42 smithi088 bash[68960]: cluster 2024-09-13T00:43:40.438618+0000 mgr.y (mgr.25304) 214 : cluster [DBG] pgmap v379: 105 pgs: 32 active+undersized, 1 stale+active+clean, 11 active+undersized+degraded, 61 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s rd, 5.0 KiB/s wr, 82 op/s; 13/168 objects degraded (7.738%) 2024-09-13T00:43:42.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:42 smithi088 bash[68960]: cluster 2024-09-13T00:43:41.161812+0000 mon.a (mon.0) 615 : cluster [WRN] Health check failed: Degraded data redundancy: 13/168 objects degraded (7.738%), 11 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:42 smithi088 bash[68960]: cluster 2024-09-13T00:43:41.916195+0000 mon.a (mon.0) 616 : cluster [DBG] osdmap e527: 8 total, 7 up, 8 in 2024-09-13T00:43:42.952 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.FlattenLocksImage (2013 ms) 2024-09-13T00:43:42.953 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.FlattenFailsToLockImage 2024-09-13T00:43:44.028 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:43 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-13T00:43:44.028 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:43 smithi077 systemd[1]: Stopped Ceph osd.1 for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:43:44.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:43 smithi077 bash[80681]: cluster 2024-09-13T00:43:42.439069+0000 mgr.y (mgr.25304) 215 : cluster [DBG] pgmap v382: 105 pgs: 36 active+undersized, 13 active+undersized+degraded, 56 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 70 KiB/s rd, 6.0 KiB/s wr, 99 op/s; 16/177 objects degraded (9.040%) 2024-09-13T00:43:44.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:43 smithi077 bash[80681]: audit 2024-09-13T00:43:42.683167+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:44.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:43 smithi077 bash[80681]: cluster 2024-09-13T00:43:42.920892+0000 mon.a (mon.0) 618 : cluster [DBG] osdmap e528: 8 total, 7 up, 8 in 2024-09-13T00:43:44.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:43 smithi077 bash[80681]: audit 2024-09-13T00:43:43.418375+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:44.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:43 smithi077 bash[80681]: audit 2024-09-13T00:43:43.420392+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:44.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:43 smithi077 bash[83127]: cluster 2024-09-13T00:43:42.439069+0000 mgr.y (mgr.25304) 215 : cluster [DBG] pgmap v382: 105 pgs: 36 active+undersized, 13 active+undersized+degraded, 56 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 70 KiB/s rd, 6.0 KiB/s wr, 99 op/s; 16/177 objects degraded (9.040%) 2024-09-13T00:43:44.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:43 smithi077 bash[83127]: audit 2024-09-13T00:43:42.683167+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:44.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:43 smithi077 bash[83127]: cluster 2024-09-13T00:43:42.920892+0000 mon.a (mon.0) 618 : cluster [DBG] osdmap e528: 8 total, 7 up, 8 in 2024-09-13T00:43:44.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:43 smithi077 bash[83127]: audit 2024-09-13T00:43:43.418375+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:44.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:43 smithi077 bash[83127]: audit 2024-09-13T00:43:43.420392+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:44.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:43 smithi088 bash[68960]: cluster 2024-09-13T00:43:42.439069+0000 mgr.y (mgr.25304) 215 : cluster [DBG] pgmap v382: 105 pgs: 36 active+undersized, 13 active+undersized+degraded, 56 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 70 KiB/s rd, 6.0 KiB/s wr, 99 op/s; 16/177 objects degraded (9.040%) 2024-09-13T00:43:44.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:43 smithi088 bash[68960]: audit 2024-09-13T00:43:42.683167+0000 mon.a (mon.0) 617 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:44.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:43 smithi088 bash[68960]: cluster 2024-09-13T00:43:42.920892+0000 mon.a (mon.0) 618 : cluster [DBG] osdmap e528: 8 total, 7 up, 8 in 2024-09-13T00:43:44.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:43 smithi088 bash[68960]: audit 2024-09-13T00:43:43.418375+0000 mon.a (mon.0) 619 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:44.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:43 smithi088 bash[68960]: audit 2024-09-13T00:43:43.420392+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:44.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:43:44 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:44.528 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:43:44 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:44.529 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:43:44 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:44.529 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:43:44 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:44.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:44 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:44.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:44 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:44.530 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:43:44 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:44.530 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:43:44 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:44.530 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:44 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:43:44.530 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:44 smithi077 systemd[1]: Started Ceph osd.1 for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:43:44.960 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.FlattenFailsToLockImage (2008 ms) 2024-09-13T00:43:44.960 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklisted 2024-09-13T00:43:45.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:44 smithi077 bash[80681]: cluster 2024-09-13T00:43:43.921441+0000 mon.a (mon.0) 621 : cluster [DBG] osdmap e529: 8 total, 7 up, 8 in 2024-09-13T00:43:45.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:44 smithi077 bash[80681]: audit 2024-09-13T00:43:44.189682+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:45.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:44 smithi077 bash[80681]: audit 2024-09-13T00:43:44.197400+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:45.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:44 smithi077 bash[80681]: audit 2024-09-13T00:43:44.198500+0000 mon.a (mon.0) 624 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:43:45.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:44 smithi077 bash[83127]: cluster 2024-09-13T00:43:43.921441+0000 mon.a (mon.0) 621 : cluster [DBG] osdmap e529: 8 total, 7 up, 8 in 2024-09-13T00:43:45.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:44 smithi077 bash[83127]: audit 2024-09-13T00:43:44.189682+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:45.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:44 smithi077 bash[83127]: audit 2024-09-13T00:43:44.197400+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:45.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:44 smithi077 bash[83127]: audit 2024-09-13T00:43:44.198500+0000 mon.a (mon.0) 624 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:43:45.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:44 smithi088 bash[68960]: cluster 2024-09-13T00:43:43.921441+0000 mon.a (mon.0) 621 : cluster [DBG] osdmap e529: 8 total, 7 up, 8 in 2024-09-13T00:43:45.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:44 smithi088 bash[68960]: audit 2024-09-13T00:43:44.189682+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:45.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:44 smithi088 bash[68960]: audit 2024-09-13T00:43:44.197400+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:45.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:44 smithi088 bash[68960]: audit 2024-09-13T00:43:44.198500+0000 mon.a (mon.0) 624 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:43:45.926 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklisted (964 ms) 2024-09-13T00:43:45.926 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklistedWatch 2024-09-13T00:43:46.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:45 smithi077 bash[80681]: cluster 2024-09-13T00:43:44.440393+0000 mgr.y (mgr.25304) 216 : cluster [DBG] pgmap v385: 105 pgs: 35 active+undersized, 14 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 107 KiB/s rd, 9.2 KiB/s wr, 152 op/s; 22/204 objects degraded (10.784%) 2024-09-13T00:43:46.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:45 smithi077 bash[80681]: cluster 2024-09-13T00:43:44.931614+0000 mon.a (mon.0) 625 : cluster [DBG] osdmap e530: 8 total, 7 up, 8 in 2024-09-13T00:43:46.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:45 smithi077 bash[80681]: audit 2024-09-13T00:43:45.051487+0000 mon.a (mon.0) 626 : audit [INF] from='client.? 172.21.15.77:0/3073421762' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/3073421762"}]: dispatch 2024-09-13T00:43:46.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:45 smithi077 bash[80681]: audit 2024-09-13T00:43:45.917638+0000 mon.a (mon.0) 627 : audit [INF] from='client.? 172.21.15.77:0/3073421762' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/3073421762"}]': finished 2024-09-13T00:43:46.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:45 smithi077 bash[80681]: cluster 2024-09-13T00:43:45.917790+0000 mon.a (mon.0) 628 : cluster [DBG] osdmap e531: 8 total, 7 up, 8 in 2024-09-13T00:43:46.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:45 smithi077 bash[83127]: cluster 2024-09-13T00:43:44.440393+0000 mgr.y (mgr.25304) 216 : cluster [DBG] pgmap v385: 105 pgs: 35 active+undersized, 14 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 107 KiB/s rd, 9.2 KiB/s wr, 152 op/s; 22/204 objects degraded (10.784%) 2024-09-13T00:43:46.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:45 smithi077 bash[83127]: cluster 2024-09-13T00:43:44.931614+0000 mon.a (mon.0) 625 : cluster [DBG] osdmap e530: 8 total, 7 up, 8 in 2024-09-13T00:43:46.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:45 smithi077 bash[83127]: audit 2024-09-13T00:43:45.051487+0000 mon.a (mon.0) 626 : audit [INF] from='client.? 172.21.15.77:0/3073421762' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/3073421762"}]: dispatch 2024-09-13T00:43:46.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:45 smithi077 bash[83127]: audit 2024-09-13T00:43:45.917638+0000 mon.a (mon.0) 627 : audit [INF] from='client.? 172.21.15.77:0/3073421762' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/3073421762"}]': finished 2024-09-13T00:43:46.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:45 smithi077 bash[83127]: cluster 2024-09-13T00:43:45.917790+0000 mon.a (mon.0) 628 : cluster [DBG] osdmap e531: 8 total, 7 up, 8 in 2024-09-13T00:43:46.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:45 smithi088 bash[68960]: cluster 2024-09-13T00:43:44.440393+0000 mgr.y (mgr.25304) 216 : cluster [DBG] pgmap v385: 105 pgs: 35 active+undersized, 14 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 107 KiB/s rd, 9.2 KiB/s wr, 152 op/s; 22/204 objects degraded (10.784%) 2024-09-13T00:43:46.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:45 smithi088 bash[68960]: cluster 2024-09-13T00:43:44.931614+0000 mon.a (mon.0) 625 : cluster [DBG] osdmap e530: 8 total, 7 up, 8 in 2024-09-13T00:43:46.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:45 smithi088 bash[68960]: audit 2024-09-13T00:43:45.051487+0000 mon.a (mon.0) 626 : audit [INF] from='client.? 172.21.15.77:0/3073421762' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/3073421762"}]: dispatch 2024-09-13T00:43:46.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:45 smithi088 bash[68960]: audit 2024-09-13T00:43:45.917638+0000 mon.a (mon.0) 627 : audit [INF] from='client.? 172.21.15.77:0/3073421762' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/3073421762"}]': finished 2024-09-13T00:43:46.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:45 smithi088 bash[68960]: cluster 2024-09-13T00:43:45.917790+0000 mon.a (mon.0) 628 : cluster [DBG] osdmap e531: 8 total, 7 up, 8 in 2024-09-13T00:43:47.252 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:46 smithi077 bash[83127]: audit 2024-09-13T00:43:46.001371+0000 mon.c (mon.2) 19 : audit [INF] from='client.? 172.21.15.77:0/1338001532' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/1338001532"}]: dispatch 2024-09-13T00:43:47.252 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:46 smithi077 bash[83127]: audit 2024-09-13T00:43:46.001728+0000 mon.a (mon.0) 629 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/1338001532"}]: dispatch 2024-09-13T00:43:47.253 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:46 smithi077 bash[80681]: audit 2024-09-13T00:43:46.001371+0000 mon.c (mon.2) 19 : audit [INF] from='client.? 172.21.15.77:0/1338001532' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/1338001532"}]: dispatch 2024-09-13T00:43:47.253 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:46 smithi077 bash[80681]: audit 2024-09-13T00:43:46.001728+0000 mon.a (mon.0) 629 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/1338001532"}]: dispatch 2024-09-13T00:43:47.253 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:47 smithi077 bash[94412]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T00:43:47.253 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:47 smithi077 bash[94412]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-13T00:43:47.253 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:47 smithi077 bash[94412]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-13T00:43:47.253 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:47 smithi077 bash[94412]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-13T00:43:47.253 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:47 smithi077 bash[94412]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-13T00:43:47.253 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:47 smithi077 bash[94412]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-13T00:43:47.253 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:47 smithi077 bash[94412]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-13T00:43:47.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:46 smithi088 bash[68960]: audit 2024-09-13T00:43:46.001371+0000 mon.c (mon.2) 19 : audit [INF] from='client.? 172.21.15.77:0/1338001532' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/1338001532"}]: dispatch 2024-09-13T00:43:47.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:46 smithi088 bash[68960]: audit 2024-09-13T00:43:46.001728+0000 mon.a (mon.0) 629 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/1338001532"}]: dispatch 2024-09-13T00:43:47.950 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklistedWatch (2025 ms) 2024-09-13T00:43:47.950 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.AioWriteRequestsLock 2024-09-13T00:43:48.018 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.AioWriteRequestsLock (69 ms) 2024-09-13T00:43:48.019 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.AioDiscardRequestsLock 2024-09-13T00:43:48.081 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.AioDiscardRequestsLock (62 ms) 2024-09-13T00:43:48.081 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.CancelAsyncResize 2024-09-13T00:43:48.202 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.CancelAsyncResize (122 ms) 2024-09-13T00:43:48.202 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.MultipleResize 2024-09-13T00:43:48.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:47 smithi077 bash[80681]: cluster 2024-09-13T00:43:46.441467+0000 mgr.y (mgr.25304) 217 : cluster [DBG] pgmap v388: 105 pgs: 34 active+undersized, 15 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 12 KiB/s wr, 256 op/s; 23/207 objects degraded (11.111%) 2024-09-13T00:43:48.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:47 smithi077 bash[80681]: cluster 2024-09-13T00:43:46.937634+0000 mon.a (mon.0) 630 : cluster [WRN] Health check update: Degraded data redundancy: 23/207 objects degraded (11.111%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:48.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:47 smithi077 bash[80681]: audit 2024-09-13T00:43:46.948547+0000 mon.a (mon.0) 631 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/1338001532"}]': finished 2024-09-13T00:43:48.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:47 smithi077 bash[80681]: cluster 2024-09-13T00:43:46.948634+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e532: 8 total, 7 up, 8 in 2024-09-13T00:43:48.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:47 smithi077 bash[83127]: cluster 2024-09-13T00:43:46.441467+0000 mgr.y (mgr.25304) 217 : cluster [DBG] pgmap v388: 105 pgs: 34 active+undersized, 15 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 12 KiB/s wr, 256 op/s; 23/207 objects degraded (11.111%) 2024-09-13T00:43:48.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:47 smithi077 bash[83127]: cluster 2024-09-13T00:43:46.937634+0000 mon.a (mon.0) 630 : cluster [WRN] Health check update: Degraded data redundancy: 23/207 objects degraded (11.111%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:48.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:47 smithi077 bash[83127]: audit 2024-09-13T00:43:46.948547+0000 mon.a (mon.0) 631 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/1338001532"}]': finished 2024-09-13T00:43:48.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:47 smithi077 bash[83127]: cluster 2024-09-13T00:43:46.948634+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e532: 8 total, 7 up, 8 in 2024-09-13T00:43:48.349 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.MultipleResize (147 ms) 2024-09-13T00:43:48.349 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.Metadata 2024-09-13T00:43:48.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:47 smithi088 bash[68960]: cluster 2024-09-13T00:43:46.441467+0000 mgr.y (mgr.25304) 217 : cluster [DBG] pgmap v388: 105 pgs: 34 active+undersized, 15 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 12 KiB/s wr, 256 op/s; 23/207 objects degraded (11.111%) 2024-09-13T00:43:48.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:47 smithi088 bash[68960]: cluster 2024-09-13T00:43:46.937634+0000 mon.a (mon.0) 630 : cluster [WRN] Health check update: Degraded data redundancy: 23/207 objects degraded (11.111%), 15 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:48.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:47 smithi088 bash[68960]: audit 2024-09-13T00:43:46.948547+0000 mon.a (mon.0) 631 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/1338001532"}]': finished 2024-09-13T00:43:48.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:47 smithi088 bash[68960]: cluster 2024-09-13T00:43:46.948634+0000 mon.a (mon.0) 632 : cluster [DBG] osdmap e532: 8 total, 7 up, 8 in 2024-09-13T00:43:48.467 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.Metadata (117 ms) 2024-09-13T00:43:48.467 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.MetadataConfApply 2024-09-13T00:43:48.548 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.MetadataConfApply (81 ms) 2024-09-13T00:43:48.548 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.SnapshotCopyup 2024-09-13T00:43:48.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:43:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:43:48] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2024-09-13T00:43:49.778 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:49 smithi077 bash[94733]: debug 2024-09-13T00:43:49.458+0000 7f134a7c7640 1 -- 172.21.15.77:0/3825493250 <== mon.2 v2:172.21.15.77:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5647220e31e0 con 0x5647220a3800 2024-09-13T00:43:50.278 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:50 smithi077 bash[94733]: debug 2024-09-13T00:43:50.018+0000 7f134d02d640 -1 Falling back to public interface 2024-09-13T00:43:50.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:49 smithi077 bash[83127]: cluster 2024-09-13T00:43:48.442456+0000 mgr.y (mgr.25304) 218 : cluster [DBG] pgmap v390: 105 pgs: 34 active+undersized, 15 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 169 KiB/s rd, 11 KiB/s wr, 227 op/s; 23/207 objects degraded (11.111%) 2024-09-13T00:43:50.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:49 smithi077 bash[83127]: cluster 2024-09-13T00:43:48.959131+0000 mon.a (mon.0) 633 : cluster [DBG] osdmap e533: 8 total, 7 up, 8 in 2024-09-13T00:43:50.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:49 smithi077 bash[80681]: cluster 2024-09-13T00:43:48.442456+0000 mgr.y (mgr.25304) 218 : cluster [DBG] pgmap v390: 105 pgs: 34 active+undersized, 15 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 169 KiB/s rd, 11 KiB/s wr, 227 op/s; 23/207 objects degraded (11.111%) 2024-09-13T00:43:50.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:49 smithi077 bash[80681]: cluster 2024-09-13T00:43:48.959131+0000 mon.a (mon.0) 633 : cluster [DBG] osdmap e533: 8 total, 7 up, 8 in 2024-09-13T00:43:50.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:49 smithi088 bash[68960]: cluster 2024-09-13T00:43:48.442456+0000 mgr.y (mgr.25304) 218 : cluster [DBG] pgmap v390: 105 pgs: 34 active+undersized, 15 active+undersized+degraded, 56 active+clean; 583 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 169 KiB/s rd, 11 KiB/s wr, 227 op/s; 23/207 objects degraded (11.111%) 2024-09-13T00:43:50.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:49 smithi088 bash[68960]: cluster 2024-09-13T00:43:48.959131+0000 mon.a (mon.0) 633 : cluster [DBG] osdmap e533: 8 total, 7 up, 8 in 2024-09-13T00:43:51.090 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.SnapshotCopyup (2542 ms) 2024-09-13T00:43:51.090 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.SnapshotCopyupZeros 2024-09-13T00:43:51.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:50 smithi077 bash[80681]: cluster 2024-09-13T00:43:49.958143+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e534: 8 total, 7 up, 8 in 2024-09-13T00:43:51.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:50 smithi077 bash[80681]: audit 2024-09-13T00:43:50.018353+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:51.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:50 smithi077 bash[80681]: audit 2024-09-13T00:43:50.026305+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:51.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:50 smithi077 bash[80681]: audit 2024-09-13T00:43:50.690528+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:51.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:50 smithi077 bash[80681]: audit 2024-09-13T00:43:50.696701+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:51.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:50 smithi077 bash[83127]: cluster 2024-09-13T00:43:49.958143+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e534: 8 total, 7 up, 8 in 2024-09-13T00:43:51.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:50 smithi077 bash[83127]: audit 2024-09-13T00:43:50.018353+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:51.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:50 smithi077 bash[83127]: audit 2024-09-13T00:43:50.026305+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:51.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:50 smithi077 bash[83127]: audit 2024-09-13T00:43:50.690528+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:51.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:50 smithi077 bash[83127]: audit 2024-09-13T00:43:50.696701+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:51.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:50 smithi088 bash[68960]: cluster 2024-09-13T00:43:49.958143+0000 mon.a (mon.0) 634 : cluster [DBG] osdmap e534: 8 total, 7 up, 8 in 2024-09-13T00:43:51.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:50 smithi088 bash[68960]: audit 2024-09-13T00:43:50.018353+0000 mon.a (mon.0) 635 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:51.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:50 smithi088 bash[68960]: audit 2024-09-13T00:43:50.026305+0000 mon.a (mon.0) 636 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:51.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:50 smithi088 bash[68960]: audit 2024-09-13T00:43:50.690528+0000 mon.a (mon.0) 637 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:51.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:50 smithi088 bash[68960]: audit 2024-09-13T00:43:50.696701+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:52.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:51 smithi077 bash[80681]: cluster 2024-09-13T00:43:50.443109+0000 mgr.y (mgr.25304) 219 : cluster [DBG] pgmap v393: 105 pgs: 32 active+undersized, 17 active+undersized+degraded, 56 active+clean; 585 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 200 KiB/s rd, 22 KiB/s wr, 283 op/s; 40/294 objects degraded (13.605%) 2024-09-13T00:43:52.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:51 smithi077 bash[80681]: cluster 2024-09-13T00:43:50.972105+0000 mon.a (mon.0) 639 : cluster [DBG] osdmap e535: 8 total, 7 up, 8 in 2024-09-13T00:43:52.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:51 smithi077 bash[83127]: cluster 2024-09-13T00:43:50.443109+0000 mgr.y (mgr.25304) 219 : cluster [DBG] pgmap v393: 105 pgs: 32 active+undersized, 17 active+undersized+degraded, 56 active+clean; 585 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 200 KiB/s rd, 22 KiB/s wr, 283 op/s; 40/294 objects degraded (13.605%) 2024-09-13T00:43:52.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:51 smithi077 bash[83127]: cluster 2024-09-13T00:43:50.972105+0000 mon.a (mon.0) 639 : cluster [DBG] osdmap e535: 8 total, 7 up, 8 in 2024-09-13T00:43:52.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:51 smithi088 bash[68960]: cluster 2024-09-13T00:43:50.443109+0000 mgr.y (mgr.25304) 219 : cluster [DBG] pgmap v393: 105 pgs: 32 active+undersized, 17 active+undersized+degraded, 56 active+clean; 585 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 200 KiB/s rd, 22 KiB/s wr, 283 op/s; 40/294 objects degraded (13.605%) 2024-09-13T00:43:52.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:51 smithi088 bash[68960]: cluster 2024-09-13T00:43:50.972105+0000 mon.a (mon.0) 639 : cluster [DBG] osdmap e535: 8 total, 7 up, 8 in 2024-09-13T00:43:52.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:43:52 smithi088 bash[61791]: ts=2024-09-13T00:43:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:43:53.073 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.SnapshotCopyupZeros (1983 ms) 2024-09-13T00:43:53.073 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.SnapshotCopyupZerosMigration 2024-09-13T00:43:53.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:53 smithi088 bash[68960]: cluster 2024-09-13T00:43:52.031210+0000 mon.a (mon.0) 640 : cluster [DBG] osdmap e536: 8 total, 7 up, 8 in 2024-09-13T00:43:53.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:53 smithi077 bash[80681]: cluster 2024-09-13T00:43:52.031210+0000 mon.a (mon.0) 640 : cluster [DBG] osdmap e536: 8 total, 7 up, 8 in 2024-09-13T00:43:53.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:53 smithi077 bash[83127]: cluster 2024-09-13T00:43:52.031210+0000 mon.a (mon.0) 640 : cluster [DBG] osdmap e536: 8 total, 7 up, 8 in 2024-09-13T00:43:53.528 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:53 smithi077 bash[94733]: debug 2024-09-13T00:43:53.058+0000 7f134d02d640 -1 osd.1 522 log_to_monitors true 2024-09-13T00:43:54.093 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.SnapshotCopyupZerosMigration (1020 ms) 2024-09-13T00:43:54.093 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.ResizeCopyup 2024-09-13T00:43:54.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:54 smithi088 bash[68960]: cluster 2024-09-13T00:43:52.443852+0000 mgr.y (mgr.25304) 220 : cluster [DBG] pgmap v396: 105 pgs: 31 active+undersized, 18 active+undersized+degraded, 56 active+clean; 586 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 184 KiB/s rd, 28 KiB/s wr, 270 op/s; 41/312 objects degraded (13.141%) 2024-09-13T00:43:54.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:54 smithi088 bash[68960]: cluster 2024-09-13T00:43:53.026893+0000 mon.a (mon.0) 641 : cluster [WRN] Health check update: Degraded data redundancy: 41/312 objects degraded (13.141%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:54.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:54 smithi088 bash[68960]: cluster 2024-09-13T00:43:53.039156+0000 mon.a (mon.0) 642 : cluster [DBG] osdmap e537: 8 total, 7 up, 8 in 2024-09-13T00:43:54.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:54 smithi088 bash[68960]: audit 2024-09-13T00:43:53.070078+0000 mon.c (mon.2) 20 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/975728159,v1:172.21.15.77:6811/975728159]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T00:43:54.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:54 smithi088 bash[68960]: audit 2024-09-13T00:43:53.070400+0000 mon.a (mon.0) 643 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T00:43:54.528 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:43:54 smithi077 bash[94733]: debug 2024-09-13T00:43:54.086+0000 7f1344dd7640 -1 osd.1 522 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T00:43:54.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:54 smithi077 bash[80681]: cluster 2024-09-13T00:43:52.443852+0000 mgr.y (mgr.25304) 220 : cluster [DBG] pgmap v396: 105 pgs: 31 active+undersized, 18 active+undersized+degraded, 56 active+clean; 586 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 184 KiB/s rd, 28 KiB/s wr, 270 op/s; 41/312 objects degraded (13.141%) 2024-09-13T00:43:54.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:54 smithi077 bash[80681]: cluster 2024-09-13T00:43:53.026893+0000 mon.a (mon.0) 641 : cluster [WRN] Health check update: Degraded data redundancy: 41/312 objects degraded (13.141%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:54 smithi077 bash[80681]: cluster 2024-09-13T00:43:53.039156+0000 mon.a (mon.0) 642 : cluster [DBG] osdmap e537: 8 total, 7 up, 8 in 2024-09-13T00:43:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:54 smithi077 bash[80681]: audit 2024-09-13T00:43:53.070078+0000 mon.c (mon.2) 20 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/975728159,v1:172.21.15.77:6811/975728159]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T00:43:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:54 smithi077 bash[80681]: audit 2024-09-13T00:43:53.070400+0000 mon.a (mon.0) 643 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T00:43:54.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:54 smithi077 bash[83127]: cluster 2024-09-13T00:43:52.443852+0000 mgr.y (mgr.25304) 220 : cluster [DBG] pgmap v396: 105 pgs: 31 active+undersized, 18 active+undersized+degraded, 56 active+clean; 586 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 184 KiB/s rd, 28 KiB/s wr, 270 op/s; 41/312 objects degraded (13.141%) 2024-09-13T00:43:54.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:54 smithi077 bash[83127]: cluster 2024-09-13T00:43:53.026893+0000 mon.a (mon.0) 641 : cluster [WRN] Health check update: Degraded data redundancy: 41/312 objects degraded (13.141%), 18 pgs degraded (PG_DEGRADED) 2024-09-13T00:43:54.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:54 smithi077 bash[83127]: cluster 2024-09-13T00:43:53.039156+0000 mon.a (mon.0) 642 : cluster [DBG] osdmap e537: 8 total, 7 up, 8 in 2024-09-13T00:43:54.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:54 smithi077 bash[83127]: audit 2024-09-13T00:43:53.070078+0000 mon.c (mon.2) 20 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/975728159,v1:172.21.15.77:6811/975728159]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T00:43:54.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:54 smithi077 bash[83127]: audit 2024-09-13T00:43:53.070400+0000 mon.a (mon.0) 643 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-13T00:43:55.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:55 smithi088 bash[68960]: audit 2024-09-13T00:43:54.044682+0000 mon.a (mon.0) 644 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T00:43:55.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:55 smithi088 bash[68960]: cluster 2024-09-13T00:43:54.044941+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e538: 8 total, 7 up, 8 in 2024-09-13T00:43:55.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:55 smithi088 bash[68960]: audit 2024-09-13T00:43:54.047920+0000 mon.c (mon.2) 21 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/975728159,v1:172.21.15.77:6811/975728159]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:43:55.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:55 smithi088 bash[68960]: audit 2024-09-13T00:43:54.054803+0000 mon.a (mon.0) 646 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:43:55.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:55 smithi077 bash[80681]: debug 2024-09-13T00:43:55.046+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:43:55.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:55 smithi077 bash[80681]: audit 2024-09-13T00:43:54.044682+0000 mon.a (mon.0) 644 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T00:43:55.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:55 smithi077 bash[80681]: cluster 2024-09-13T00:43:54.044941+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e538: 8 total, 7 up, 8 in 2024-09-13T00:43:55.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:55 smithi077 bash[80681]: audit 2024-09-13T00:43:54.047920+0000 mon.c (mon.2) 21 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/975728159,v1:172.21.15.77:6811/975728159]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:43:55.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:55 smithi077 bash[80681]: audit 2024-09-13T00:43:54.054803+0000 mon.a (mon.0) 646 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:43:55.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:55 smithi077 bash[83127]: audit 2024-09-13T00:43:54.044682+0000 mon.a (mon.0) 644 : audit [INF] from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-13T00:43:55.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:55 smithi077 bash[83127]: cluster 2024-09-13T00:43:54.044941+0000 mon.a (mon.0) 645 : cluster [DBG] osdmap e538: 8 total, 7 up, 8 in 2024-09-13T00:43:55.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:55 smithi077 bash[83127]: audit 2024-09-13T00:43:54.047920+0000 mon.c (mon.2) 21 : audit [INF] from='osd.1 [v2:172.21.15.77:6810/975728159,v1:172.21.15.77:6811/975728159]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:43:55.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:55 smithi077 bash[83127]: audit 2024-09-13T00:43:54.054803+0000 mon.a (mon.0) 646 : audit [INF] from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:43:56.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:56 smithi088 bash[68960]: cluster 2024-09-13T00:43:54.445225+0000 mgr.y (mgr.25304) 221 : cluster [DBG] pgmap v399: 105 pgs: 30 active+undersized, 19 active+undersized+degraded, 56 active+clean; 604 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 245 KiB/s rd, 25 KiB/s wr, 331 op/s; 56/384 objects degraded (14.583%) 2024-09-13T00:43:56.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:56 smithi088 bash[68960]: cluster 2024-09-13T00:43:55.051243+0000 mon.a (mon.0) 647 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:43:56.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:56 smithi088 bash[68960]: cluster 2024-09-13T00:43:55.051318+0000 mon.a (mon.0) 648 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T00:43:56.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:56 smithi088 bash[68960]: cluster 2024-09-13T00:43:55.066369+0000 mon.a (mon.0) 649 : cluster [INF] osd.1 [v2:172.21.15.77:6810/975728159,v1:172.21.15.77:6811/975728159] boot 2024-09-13T00:43:56.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:56 smithi088 bash[68960]: cluster 2024-09-13T00:43:55.066513+0000 mon.a (mon.0) 650 : cluster [DBG] osdmap e539: 8 total, 8 up, 8 in 2024-09-13T00:43:56.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:56 smithi088 bash[68960]: audit 2024-09-13T00:43:55.068923+0000 mon.a (mon.0) 651 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:43:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:56 smithi077 bash[80681]: cluster 2024-09-13T00:43:54.445225+0000 mgr.y (mgr.25304) 221 : cluster [DBG] pgmap v399: 105 pgs: 30 active+undersized, 19 active+undersized+degraded, 56 active+clean; 604 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 245 KiB/s rd, 25 KiB/s wr, 331 op/s; 56/384 objects degraded (14.583%) 2024-09-13T00:43:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:56 smithi077 bash[80681]: cluster 2024-09-13T00:43:55.051243+0000 mon.a (mon.0) 647 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:43:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:56 smithi077 bash[80681]: cluster 2024-09-13T00:43:55.051318+0000 mon.a (mon.0) 648 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T00:43:56.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:56 smithi077 bash[80681]: cluster 2024-09-13T00:43:55.066369+0000 mon.a (mon.0) 649 : cluster [INF] osd.1 [v2:172.21.15.77:6810/975728159,v1:172.21.15.77:6811/975728159] boot 2024-09-13T00:43:56.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:56 smithi077 bash[80681]: cluster 2024-09-13T00:43:55.066513+0000 mon.a (mon.0) 650 : cluster [DBG] osdmap e539: 8 total, 8 up, 8 in 2024-09-13T00:43:56.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:56 smithi077 bash[80681]: audit 2024-09-13T00:43:55.068923+0000 mon.a (mon.0) 651 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:43:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:56 smithi077 bash[83127]: cluster 2024-09-13T00:43:54.445225+0000 mgr.y (mgr.25304) 221 : cluster [DBG] pgmap v399: 105 pgs: 30 active+undersized, 19 active+undersized+degraded, 56 active+clean; 604 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 245 KiB/s rd, 25 KiB/s wr, 331 op/s; 56/384 objects degraded (14.583%) 2024-09-13T00:43:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:56 smithi077 bash[83127]: cluster 2024-09-13T00:43:55.051243+0000 mon.a (mon.0) 647 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:43:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:56 smithi077 bash[83127]: cluster 2024-09-13T00:43:55.051318+0000 mon.a (mon.0) 648 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T00:43:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:56 smithi077 bash[83127]: cluster 2024-09-13T00:43:55.066369+0000 mon.a (mon.0) 649 : cluster [INF] osd.1 [v2:172.21.15.77:6810/975728159,v1:172.21.15.77:6811/975728159] boot 2024-09-13T00:43:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:56 smithi077 bash[83127]: cluster 2024-09-13T00:43:55.066513+0000 mon.a (mon.0) 650 : cluster [DBG] osdmap e539: 8 total, 8 up, 8 in 2024-09-13T00:43:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:56 smithi077 bash[83127]: audit 2024-09-13T00:43:55.068923+0000 mon.a (mon.0) 651 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-13T00:43:57.206 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.ResizeCopyup (3114 ms) 2024-09-13T00:43:57.207 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.DiscardCopyup 2024-09-13T00:43:57.229 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:43:57.229 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.DiscardCopyup (22 ms) 2024-09-13T00:43:57.229 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.ImageOptions 2024-09-13T00:43:57.250 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.ImageOptions (22 ms) 2024-09-13T00:43:57.251 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.WriteFullCopyup 2024-09-13T00:43:57.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:57 smithi088 bash[68960]: cluster 2024-09-13T00:43:56.071842+0000 mon.a (mon.0) 652 : cluster [DBG] osdmap e540: 8 total, 8 up, 8 in 2024-09-13T00:43:57.352 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:57 smithi077 bash[80681]: cluster 2024-09-13T00:43:56.071842+0000 mon.a (mon.0) 652 : cluster [DBG] osdmap e540: 8 total, 8 up, 8 in 2024-09-13T00:43:57.352 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:57 smithi077 bash[83127]: cluster 2024-09-13T00:43:56.071842+0000 mon.a (mon.0) 652 : cluster [DBG] osdmap e540: 8 total, 8 up, 8 in 2024-09-13T00:43:58.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:58 smithi088 bash[68960]: cluster 2024-09-13T00:43:56.446542+0000 mgr.y (mgr.25304) 222 : cluster [DBG] pgmap v402: 105 pgs: 2 active+recovery_wait+degraded, 29 active+undersized, 15 active+undersized+degraded, 59 active+clean; 604 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 218 KiB/s rd, 22 KiB/s wr, 298 op/s; 56/399 objects degraded (14.035%); 3 B/s, 0 objects/s recovering 2024-09-13T00:43:58.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:58 smithi088 bash[68960]: cluster 2024-09-13T00:43:57.081316+0000 mon.a (mon.0) 653 : cluster [DBG] osdmap e541: 8 total, 8 up, 8 in 2024-09-13T00:43:58.431 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:58 smithi077 bash[80681]: cluster 2024-09-13T00:43:56.446542+0000 mgr.y (mgr.25304) 222 : cluster [DBG] pgmap v402: 105 pgs: 2 active+recovery_wait+degraded, 29 active+undersized, 15 active+undersized+degraded, 59 active+clean; 604 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 218 KiB/s rd, 22 KiB/s wr, 298 op/s; 56/399 objects degraded (14.035%); 3 B/s, 0 objects/s recovering 2024-09-13T00:43:58.431 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:58 smithi077 bash[80681]: cluster 2024-09-13T00:43:57.081316+0000 mon.a (mon.0) 653 : cluster [DBG] osdmap e541: 8 total, 8 up, 8 in 2024-09-13T00:43:58.432 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:58 smithi077 bash[83127]: cluster 2024-09-13T00:43:56.446542+0000 mgr.y (mgr.25304) 222 : cluster [DBG] pgmap v402: 105 pgs: 2 active+recovery_wait+degraded, 29 active+undersized, 15 active+undersized+degraded, 59 active+clean; 604 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 218 KiB/s rd, 22 KiB/s wr, 298 op/s; 56/399 objects degraded (14.035%); 3 B/s, 0 objects/s recovering 2024-09-13T00:43:58.432 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:58 smithi077 bash[83127]: cluster 2024-09-13T00:43:57.081316+0000 mon.a (mon.0) 653 : cluster [DBG] osdmap e541: 8 total, 8 up, 8 in 2024-09-13T00:43:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:43:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:43:58] "GET /metrics HTTP/1.1" 200 36410 "" "Prometheus/2.43.0" 2024-09-13T00:43:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:59 smithi088 bash[68960]: cluster 2024-09-13T00:43:58.094436+0000 mon.a (mon.0) 654 : cluster [DBG] osdmap e542: 8 total, 8 up, 8 in 2024-09-13T00:43:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:59 smithi088 bash[68960]: audit 2024-09-13T00:43:58.421851+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:43:59 smithi088 bash[68960]: audit 2024-09-13T00:43:58.423613+0000 mon.a (mon.0) 656 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:59 smithi077 bash[83127]: cluster 2024-09-13T00:43:58.094436+0000 mon.a (mon.0) 654 : cluster [DBG] osdmap e542: 8 total, 8 up, 8 in 2024-09-13T00:43:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:59 smithi077 bash[83127]: audit 2024-09-13T00:43:58.421851+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:43:59 smithi077 bash[83127]: audit 2024-09-13T00:43:58.423613+0000 mon.a (mon.0) 656 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:43:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:59 smithi077 bash[80681]: cluster 2024-09-13T00:43:58.094436+0000 mon.a (mon.0) 654 : cluster [DBG] osdmap e542: 8 total, 8 up, 8 in 2024-09-13T00:43:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:59 smithi077 bash[80681]: audit 2024-09-13T00:43:58.421851+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:43:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:43:59 smithi077 bash[80681]: audit 2024-09-13T00:43:58.423613+0000 mon.a (mon.0) 656 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:00.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:00 smithi077 bash[80681]: cluster 2024-09-13T00:43:58.453997+0000 mgr.y (mgr.25304) 223 : cluster [DBG] pgmap v405: 105 pgs: 2 active+recovery_wait+degraded, 29 active+undersized, 15 active+undersized+degraded, 59 active+clean; 604 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.2 KiB/s wr, 36 op/s; 56/399 objects degraded (14.035%); 3 B/s, 0 objects/s recovering 2024-09-13T00:44:00.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:00 smithi077 bash[80681]: cluster 2024-09-13T00:43:59.114485+0000 mon.a (mon.0) 657 : cluster [DBG] osdmap e543: 8 total, 8 up, 8 in 2024-09-13T00:44:00.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:00 smithi077 bash[83127]: cluster 2024-09-13T00:43:58.453997+0000 mgr.y (mgr.25304) 223 : cluster [DBG] pgmap v405: 105 pgs: 2 active+recovery_wait+degraded, 29 active+undersized, 15 active+undersized+degraded, 59 active+clean; 604 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.2 KiB/s wr, 36 op/s; 56/399 objects degraded (14.035%); 3 B/s, 0 objects/s recovering 2024-09-13T00:44:00.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:00 smithi077 bash[83127]: cluster 2024-09-13T00:43:59.114485+0000 mon.a (mon.0) 657 : cluster [DBG] osdmap e543: 8 total, 8 up, 8 in 2024-09-13T00:44:00.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:00 smithi088 bash[68960]: cluster 2024-09-13T00:43:58.453997+0000 mgr.y (mgr.25304) 223 : cluster [DBG] pgmap v405: 105 pgs: 2 active+recovery_wait+degraded, 29 active+undersized, 15 active+undersized+degraded, 59 active+clean; 604 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.2 KiB/s wr, 36 op/s; 56/399 objects degraded (14.035%); 3 B/s, 0 objects/s recovering 2024-09-13T00:44:00.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:00 smithi088 bash[68960]: cluster 2024-09-13T00:43:59.114485+0000 mon.a (mon.0) 657 : cluster [DBG] osdmap e543: 8 total, 8 up, 8 in 2024-09-13T00:44:01.169 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.WriteFullCopyup (3918 ms) 2024-09-13T00:44:01.169 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.DiffIterateCloneOverwrite 2024-09-13T00:44:01.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:01 smithi077 bash[80681]: cluster 2024-09-13T00:44:00.123801+0000 mon.a (mon.0) 658 : cluster [DBG] osdmap e544: 8 total, 8 up, 8 in 2024-09-13T00:44:01.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:01 smithi077 bash[80681]: cluster 2024-09-13T00:44:00.911584+0000 mon.a (mon.0) 659 : cluster [WRN] Health check update: Degraded data redundancy: 56/399 objects degraded (14.035%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T00:44:01.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:01 smithi077 bash[83127]: cluster 2024-09-13T00:44:00.123801+0000 mon.a (mon.0) 658 : cluster [DBG] osdmap e544: 8 total, 8 up, 8 in 2024-09-13T00:44:01.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:01 smithi077 bash[83127]: cluster 2024-09-13T00:44:00.911584+0000 mon.a (mon.0) 659 : cluster [WRN] Health check update: Degraded data redundancy: 56/399 objects degraded (14.035%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T00:44:01.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:01 smithi088 bash[68960]: cluster 2024-09-13T00:44:00.123801+0000 mon.a (mon.0) 658 : cluster [DBG] osdmap e544: 8 total, 8 up, 8 in 2024-09-13T00:44:01.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:01 smithi088 bash[68960]: cluster 2024-09-13T00:44:00.911584+0000 mon.a (mon.0) 659 : cluster [WRN] Health check update: Degraded data redundancy: 56/399 objects degraded (14.035%), 17 pgs degraded (PG_DEGRADED) 2024-09-13T00:44:02.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:02 smithi077 bash[80681]: cluster 2024-09-13T00:44:00.454566+0000 mgr.y (mgr.25304) 224 : cluster [DBG] pgmap v408: 105 pgs: 2 active+recovery_wait+degraded, 103 active+clean; 13 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 3.0 MiB/s wr, 200 op/s; 5/468 objects degraded (1.068%); 2.2 KiB/s, 33 keys/s, 10 objects/s recovering 2024-09-13T00:44:02.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:02 smithi077 bash[80681]: cluster 2024-09-13T00:44:01.144183+0000 mon.a (mon.0) 660 : cluster [DBG] osdmap e545: 8 total, 8 up, 8 in 2024-09-13T00:44:02.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:02 smithi077 bash[80681]: audit 2024-09-13T00:44:01.682529+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:02.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:02 smithi077 bash[80681]: audit 2024-09-13T00:44:01.692511+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:02.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:02 smithi077 bash[80681]: audit 2024-09-13T00:44:01.694274+0000 mon.a (mon.0) 663 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:44:02.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:02 smithi077 bash[80681]: audit 2024-09-13T00:44:01.695988+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:44:02.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:02 smithi077 bash[80681]: audit 2024-09-13T00:44:01.704514+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:02.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:02 smithi077 bash[80681]: audit 2024-09-13T00:44:01.838836+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:44:02.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:02 smithi077 bash[83127]: cluster 2024-09-13T00:44:00.454566+0000 mgr.y (mgr.25304) 224 : cluster [DBG] pgmap v408: 105 pgs: 2 active+recovery_wait+degraded, 103 active+clean; 13 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 3.0 MiB/s wr, 200 op/s; 5/468 objects degraded (1.068%); 2.2 KiB/s, 33 keys/s, 10 objects/s recovering 2024-09-13T00:44:02.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:02 smithi077 bash[83127]: cluster 2024-09-13T00:44:01.144183+0000 mon.a (mon.0) 660 : cluster [DBG] osdmap e545: 8 total, 8 up, 8 in 2024-09-13T00:44:02.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:02 smithi077 bash[83127]: audit 2024-09-13T00:44:01.682529+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:02.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:02 smithi077 bash[83127]: audit 2024-09-13T00:44:01.692511+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:02.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:02 smithi077 bash[83127]: audit 2024-09-13T00:44:01.694274+0000 mon.a (mon.0) 663 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:44:02.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:02 smithi077 bash[83127]: audit 2024-09-13T00:44:01.695988+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:44:02.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:02 smithi077 bash[83127]: audit 2024-09-13T00:44:01.704514+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:02.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:02 smithi077 bash[83127]: audit 2024-09-13T00:44:01.838836+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:44:02.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:44:02 smithi088 bash[61791]: ts=2024-09-13T00:44:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:44:02.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:02 smithi088 bash[68960]: cluster 2024-09-13T00:44:00.454566+0000 mgr.y (mgr.25304) 224 : cluster [DBG] pgmap v408: 105 pgs: 2 active+recovery_wait+degraded, 103 active+clean; 13 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.1 MiB/s rd, 3.0 MiB/s wr, 200 op/s; 5/468 objects degraded (1.068%); 2.2 KiB/s, 33 keys/s, 10 objects/s recovering 2024-09-13T00:44:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:02 smithi088 bash[68960]: cluster 2024-09-13T00:44:01.144183+0000 mon.a (mon.0) 660 : cluster [DBG] osdmap e545: 8 total, 8 up, 8 in 2024-09-13T00:44:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:02 smithi088 bash[68960]: audit 2024-09-13T00:44:01.682529+0000 mon.a (mon.0) 661 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:02 smithi088 bash[68960]: audit 2024-09-13T00:44:01.692511+0000 mon.a (mon.0) 662 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:02 smithi088 bash[68960]: audit 2024-09-13T00:44:01.694274+0000 mon.a (mon.0) 663 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:44:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:02 smithi088 bash[68960]: audit 2024-09-13T00:44:01.695988+0000 mon.a (mon.0) 664 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:44:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:02 smithi088 bash[68960]: audit 2024-09-13T00:44:01.704514+0000 mon.a (mon.0) 665 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:02 smithi088 bash[68960]: audit 2024-09-13T00:44:01.838836+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:44:03.217 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.DiffIterateCloneOverwrite (2048 ms) 2024-09-13T00:44:03.217 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.TestCoR 2024-09-13T00:44:03.238 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING due to disabled rbd_copy_on_read 2024-09-13T00:44:03.239 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.TestCoR (22 ms) 2024-09-13T00:44:03.239 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.FlattenNoEmptyObjects 2024-09-13T00:44:03.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:03 smithi077 bash[80681]: cluster 2024-09-13T00:44:02.146302+0000 mon.a (mon.0) 667 : cluster [DBG] osdmap e546: 8 total, 8 up, 8 in 2024-09-13T00:44:03.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:03 smithi077 bash[83127]: cluster 2024-09-13T00:44:02.146302+0000 mon.a (mon.0) 667 : cluster [DBG] osdmap e546: 8 total, 8 up, 8 in 2024-09-13T00:44:03.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:03 smithi088 bash[68960]: cluster 2024-09-13T00:44:02.146302+0000 mon.a (mon.0) 667 : cluster [DBG] osdmap e546: 8 total, 8 up, 8 in 2024-09-13T00:44:03.835 INFO:tasks.workunit.client.0.smithi077.stdout:made parent image "image383": 4194304K (1024 * 4K) 2024-09-13T00:44:03.835 INFO:tasks.workunit.client.0.smithi077.stdout:generated random write map: 2024-09-13T00:44:03.836 INFO:tasks.workunit.client.0.smithi077.stdout: [1 , 7172 ] 2024-09-13T00:44:03.836 INFO:tasks.workunit.client.0.smithi077.stdout: [8 , 34369 ] 2024-09-13T00:44:03.836 INFO:tasks.workunit.client.0.smithi077.stdout: [13 , 53372 ] 2024-09-13T00:44:03.836 INFO:tasks.workunit.client.0.smithi077.stdout: [18 , 77019 ] 2024-09-13T00:44:03.836 INFO:tasks.workunit.client.0.smithi077.stdout: [35 , 146915 ] 2024-09-13T00:44:03.836 INFO:tasks.workunit.client.0.smithi077.stdout: [50 , 206462 ] 2024-09-13T00:44:03.836 INFO:tasks.workunit.client.0.smithi077.stdout: [58 , 241015 ] 2024-09-13T00:44:03.836 INFO:tasks.workunit.client.0.smithi077.stdout: [62 , 255903 ] 2024-09-13T00:44:03.836 INFO:tasks.workunit.client.0.smithi077.stdout: [74 , 305912 ] 2024-09-13T00:44:03.836 INFO:tasks.workunit.client.0.smithi077.stdout: [87 , 357889 ] 2024-09-13T00:44:03.836 INFO:tasks.workunit.client.0.smithi077.stdout: [103 , 424517 ] 2024-09-13T00:44:03.837 INFO:tasks.workunit.client.0.smithi077.stdout: [108 , 443145 ] 2024-09-13T00:44:03.837 INFO:tasks.workunit.client.0.smithi077.stdout: [124 , 510886 ] 2024-09-13T00:44:03.837 INFO:tasks.workunit.client.0.smithi077.stdout: [128 , 526148 ] 2024-09-13T00:44:03.837 INFO:tasks.workunit.client.0.smithi077.stdout: [136 , 558930 ] 2024-09-13T00:44:03.837 INFO:tasks.workunit.client.0.smithi077.stdout: [162 , 666607 ] 2024-09-13T00:44:03.837 INFO:tasks.workunit.client.0.smithi077.stdout: [169 , 694585 ] 2024-09-13T00:44:03.837 INFO:tasks.workunit.client.0.smithi077.stdout: [192 , 787731 ] 2024-09-13T00:44:03.837 INFO:tasks.workunit.client.0.smithi077.stdout: [200 , 822423 ] 2024-09-13T00:44:03.837 INFO:tasks.workunit.client.0.smithi077.stdout: [205 , 842176 ] 2024-09-13T00:44:03.837 INFO:tasks.workunit.client.0.smithi077.stdout: [209 , 857011 ] 2024-09-13T00:44:03.837 INFO:tasks.workunit.client.0.smithi077.stdout: [213 , 872890 ] 2024-09-13T00:44:03.837 INFO:tasks.workunit.client.0.smithi077.stdout: [217 , 890064 ] 2024-09-13T00:44:03.837 INFO:tasks.workunit.client.0.smithi077.stdout: [232 , 951627 ] 2024-09-13T00:44:03.838 INFO:tasks.workunit.client.0.smithi077.stdout: [235 , 964807 ] 2024-09-13T00:44:03.838 INFO:tasks.workunit.client.0.smithi077.stdout: [250 , 1026091 ] 2024-09-13T00:44:03.838 INFO:tasks.workunit.client.0.smithi077.stdout: [251 , 1030575 ] 2024-09-13T00:44:03.838 INFO:tasks.workunit.client.0.smithi077.stdout: [255 , 1045372 ] 2024-09-13T00:44:03.838 INFO:tasks.workunit.client.0.smithi077.stdout: [262 , 1074664 ] 2024-09-13T00:44:03.838 INFO:tasks.workunit.client.0.smithi077.stdout: [280 , 1149051 ] 2024-09-13T00:44:03.838 INFO:tasks.workunit.client.0.smithi077.stdout: [290 , 1191295 ] 2024-09-13T00:44:03.838 INFO:tasks.workunit.client.0.smithi077.stdout: [308 , 1262895 ] 2024-09-13T00:44:03.838 INFO:tasks.workunit.client.0.smithi077.stdout: [318 , 1305815 ] 2024-09-13T00:44:03.838 INFO:tasks.workunit.client.0.smithi077.stdout: [319 , 1308346 ] 2024-09-13T00:44:03.838 INFO:tasks.workunit.client.0.smithi077.stdout: [349 , 1431454 ] 2024-09-13T00:44:03.838 INFO:tasks.workunit.client.0.smithi077.stdout: [355 , 1455668 ] 2024-09-13T00:44:03.839 INFO:tasks.workunit.client.0.smithi077.stdout: [358 , 1469091 ] 2024-09-13T00:44:03.839 INFO:tasks.workunit.client.0.smithi077.stdout: [375 , 1537926 ] 2024-09-13T00:44:03.839 INFO:tasks.workunit.client.0.smithi077.stdout: [383 , 1570216 ] 2024-09-13T00:44:03.839 INFO:tasks.workunit.client.0.smithi077.stdout: [385 , 1578118 ] 2024-09-13T00:44:03.839 INFO:tasks.workunit.client.0.smithi077.stdout: [401 , 1642771 ] 2024-09-13T00:44:03.839 INFO:tasks.workunit.client.0.smithi077.stdout: [407 , 1669822 ] 2024-09-13T00:44:03.839 INFO:tasks.workunit.client.0.smithi077.stdout: [410 , 1681711 ] 2024-09-13T00:44:03.839 INFO:tasks.workunit.client.0.smithi077.stdout: [414 , 1695897 ] 2024-09-13T00:44:03.839 INFO:tasks.workunit.client.0.smithi077.stdout: [445 , 1825578 ] 2024-09-13T00:44:03.839 INFO:tasks.workunit.client.0.smithi077.stdout: [446 , 1829274 ] 2024-09-13T00:44:03.839 INFO:tasks.workunit.client.0.smithi077.stdout: [447 , 1831035 ] 2024-09-13T00:44:03.839 INFO:tasks.workunit.client.0.smithi077.stdout: [454 , 1861073 ] 2024-09-13T00:44:03.840 INFO:tasks.workunit.client.0.smithi077.stdout: [459 , 1882606 ] 2024-09-13T00:44:03.840 INFO:tasks.workunit.client.0.smithi077.stdout: [470 , 1927755 ] 2024-09-13T00:44:03.840 INFO:tasks.workunit.client.0.smithi077.stdout: [487 , 1996856 ] 2024-09-13T00:44:03.840 INFO:tasks.workunit.client.0.smithi077.stdout: [498 , 2040914 ] 2024-09-13T00:44:03.840 INFO:tasks.workunit.client.0.smithi077.stdout: [507 , 2079347 ] 2024-09-13T00:44:03.840 INFO:tasks.workunit.client.0.smithi077.stdout: [514 , 2106029 ] 2024-09-13T00:44:03.840 INFO:tasks.workunit.client.0.smithi077.stdout: [516 , 2115143 ] 2024-09-13T00:44:03.840 INFO:tasks.workunit.client.0.smithi077.stdout: [521 , 2134283 ] 2024-09-13T00:44:03.840 INFO:tasks.workunit.client.0.smithi077.stdout: [536 , 2195750 ] 2024-09-13T00:44:03.840 INFO:tasks.workunit.client.0.smithi077.stdout: [551 , 2258564 ] 2024-09-13T00:44:03.840 INFO:tasks.workunit.client.0.smithi077.stdout: [561 , 2298178 ] 2024-09-13T00:44:03.841 INFO:tasks.workunit.client.0.smithi077.stdout: [564 , 2312424 ] 2024-09-13T00:44:03.841 INFO:tasks.workunit.client.0.smithi077.stdout: [594 , 2436129 ] 2024-09-13T00:44:03.841 INFO:tasks.workunit.client.0.smithi077.stdout: [612 , 2509576 ] 2024-09-13T00:44:03.841 INFO:tasks.workunit.client.0.smithi077.stdout: [625 , 2563164 ] 2024-09-13T00:44:03.841 INFO:tasks.workunit.client.0.smithi077.stdout: [652 , 2672352 ] 2024-09-13T00:44:03.841 INFO:tasks.workunit.client.0.smithi077.stdout: [681 , 2791221 ] 2024-09-13T00:44:03.841 INFO:tasks.workunit.client.0.smithi077.stdout: [682 , 2796963 ] 2024-09-13T00:44:03.841 INFO:tasks.workunit.client.0.smithi077.stdout: [687 , 2815796 ] 2024-09-13T00:44:03.841 INFO:tasks.workunit.client.0.smithi077.stdout: [688 , 2819190 ] 2024-09-13T00:44:03.841 INFO:tasks.workunit.client.0.smithi077.stdout: [695 , 2847169 ] 2024-09-13T00:44:03.841 INFO:tasks.workunit.client.0.smithi077.stdout: [705 , 2888369 ] 2024-09-13T00:44:03.841 INFO:tasks.workunit.client.0.smithi077.stdout: [718 , 2942474 ] 2024-09-13T00:44:03.841 INFO:tasks.workunit.client.0.smithi077.stdout: [764 , 3131005 ] 2024-09-13T00:44:03.842 INFO:tasks.workunit.client.0.smithi077.stdout: [767 , 3144429 ] 2024-09-13T00:44:03.842 INFO:tasks.workunit.client.0.smithi077.stdout: [771 , 3160835 ] 2024-09-13T00:44:03.842 INFO:tasks.workunit.client.0.smithi077.stdout: [778 , 3186962 ] 2024-09-13T00:44:03.842 INFO:tasks.workunit.client.0.smithi077.stdout: [801 , 3281788 ] 2024-09-13T00:44:03.842 INFO:tasks.workunit.client.0.smithi077.stdout: [807 , 3308859 ] 2024-09-13T00:44:03.842 INFO:tasks.workunit.client.0.smithi077.stdout: [811 , 3324875 ] 2024-09-13T00:44:03.842 INFO:tasks.workunit.client.0.smithi077.stdout: [814 , 3335465 ] 2024-09-13T00:44:03.842 INFO:tasks.workunit.client.0.smithi077.stdout: [819 , 3355562 ] 2024-09-13T00:44:03.842 INFO:tasks.workunit.client.0.smithi077.stdout: [829 , 3396696 ] 2024-09-13T00:44:03.842 INFO:tasks.workunit.client.0.smithi077.stdout: [839 , 3437094 ] 2024-09-13T00:44:03.842 INFO:tasks.workunit.client.0.smithi077.stdout: [855 , 3503143 ] 2024-09-13T00:44:03.842 INFO:tasks.workunit.client.0.smithi077.stdout: [856 , 3507171 ] 2024-09-13T00:44:03.843 INFO:tasks.workunit.client.0.smithi077.stdout: [886 , 3630352 ] 2024-09-13T00:44:03.843 INFO:tasks.workunit.client.0.smithi077.stdout: [895 , 3668505 ] 2024-09-13T00:44:03.843 INFO:tasks.workunit.client.0.smithi077.stdout: [934 , 3828698 ] 2024-09-13T00:44:03.843 INFO:tasks.workunit.client.0.smithi077.stdout: [941 , 3854780 ] 2024-09-13T00:44:03.843 INFO:tasks.workunit.client.0.smithi077.stdout: [948 , 3884089 ] 2024-09-13T00:44:03.843 INFO:tasks.workunit.client.0.smithi077.stdout: [951 , 3898606 ] 2024-09-13T00:44:03.843 INFO:tasks.workunit.client.0.smithi077.stdout: [957 , 3922165 ] 2024-09-13T00:44:03.843 INFO:tasks.workunit.client.0.smithi077.stdout: [968 , 3966127 ] 2024-09-13T00:44:03.843 INFO:tasks.workunit.client.0.smithi077.stdout: [981 , 4019889 ] 2024-09-13T00:44:03.843 INFO:tasks.workunit.client.0.smithi077.stdout: [982 , 4022951 ] 2024-09-13T00:44:03.843 INFO:tasks.workunit.client.0.smithi077.stdout: [993 , 4070015 ] 2024-09-13T00:44:03.843 INFO:tasks.workunit.client.0.smithi077.stdout: [1017 , 4166127 ] 2024-09-13T00:44:03.844 INFO:tasks.workunit.client.0.smithi077.stdout:write data based on random map 2024-09-13T00:44:03.844 INFO:tasks.workunit.client.0.smithi077.stdout: write object-1 2024-09-13T00:44:03.844 INFO:tasks.workunit.client.0.smithi077.stdout: write object-8 2024-09-13T00:44:03.844 INFO:tasks.workunit.client.0.smithi077.stdout: write object-13 2024-09-13T00:44:03.844 INFO:tasks.workunit.client.0.smithi077.stdout: write object-18 2024-09-13T00:44:03.844 INFO:tasks.workunit.client.0.smithi077.stdout: write object-35 2024-09-13T00:44:03.844 INFO:tasks.workunit.client.0.smithi077.stdout: write object-50 2024-09-13T00:44:03.844 INFO:tasks.workunit.client.0.smithi077.stdout: write object-58 2024-09-13T00:44:03.844 INFO:tasks.workunit.client.0.smithi077.stdout: write object-62 2024-09-13T00:44:03.844 INFO:tasks.workunit.client.0.smithi077.stdout: write object-74 2024-09-13T00:44:03.844 INFO:tasks.workunit.client.0.smithi077.stdout: write object-87 2024-09-13T00:44:03.844 INFO:tasks.workunit.client.0.smithi077.stdout: write object-103 2024-09-13T00:44:03.844 INFO:tasks.workunit.client.0.smithi077.stdout: write object-108 2024-09-13T00:44:03.845 INFO:tasks.workunit.client.0.smithi077.stdout: write object-124 2024-09-13T00:44:03.845 INFO:tasks.workunit.client.0.smithi077.stdout: write object-128 2024-09-13T00:44:03.845 INFO:tasks.workunit.client.0.smithi077.stdout: write object-136 2024-09-13T00:44:03.845 INFO:tasks.workunit.client.0.smithi077.stdout: write object-162 2024-09-13T00:44:03.845 INFO:tasks.workunit.client.0.smithi077.stdout: write object-169 2024-09-13T00:44:03.845 INFO:tasks.workunit.client.0.smithi077.stdout: write object-192 2024-09-13T00:44:03.845 INFO:tasks.workunit.client.0.smithi077.stdout: write object-200 2024-09-13T00:44:03.845 INFO:tasks.workunit.client.0.smithi077.stdout: write object-205 2024-09-13T00:44:03.845 INFO:tasks.workunit.client.0.smithi077.stdout: write object-209 2024-09-13T00:44:03.845 INFO:tasks.workunit.client.0.smithi077.stdout: write object-213 2024-09-13T00:44:03.845 INFO:tasks.workunit.client.0.smithi077.stdout: write object-217 2024-09-13T00:44:03.845 INFO:tasks.workunit.client.0.smithi077.stdout: write object-232 2024-09-13T00:44:03.845 INFO:tasks.workunit.client.0.smithi077.stdout: write object-235 2024-09-13T00:44:03.846 INFO:tasks.workunit.client.0.smithi077.stdout: write object-250 2024-09-13T00:44:03.846 INFO:tasks.workunit.client.0.smithi077.stdout: write object-251 2024-09-13T00:44:03.846 INFO:tasks.workunit.client.0.smithi077.stdout: write object-255 2024-09-13T00:44:03.846 INFO:tasks.workunit.client.0.smithi077.stdout: write object-262 2024-09-13T00:44:03.846 INFO:tasks.workunit.client.0.smithi077.stdout: write object-280 2024-09-13T00:44:03.846 INFO:tasks.workunit.client.0.smithi077.stdout: write object-290 2024-09-13T00:44:03.846 INFO:tasks.workunit.client.0.smithi077.stdout: write object-308 2024-09-13T00:44:03.846 INFO:tasks.workunit.client.0.smithi077.stdout: write object-318 2024-09-13T00:44:03.846 INFO:tasks.workunit.client.0.smithi077.stdout: write object-319 2024-09-13T00:44:03.846 INFO:tasks.workunit.client.0.smithi077.stdout: write object-349 2024-09-13T00:44:03.846 INFO:tasks.workunit.client.0.smithi077.stdout: write object-355 2024-09-13T00:44:03.846 INFO:tasks.workunit.client.0.smithi077.stdout: write object-358 2024-09-13T00:44:03.846 INFO:tasks.workunit.client.0.smithi077.stdout: write object-375 2024-09-13T00:44:03.847 INFO:tasks.workunit.client.0.smithi077.stdout: write object-383 2024-09-13T00:44:03.847 INFO:tasks.workunit.client.0.smithi077.stdout: write object-385 2024-09-13T00:44:03.847 INFO:tasks.workunit.client.0.smithi077.stdout: write object-401 2024-09-13T00:44:03.847 INFO:tasks.workunit.client.0.smithi077.stdout: write object-407 2024-09-13T00:44:03.847 INFO:tasks.workunit.client.0.smithi077.stdout: write object-410 2024-09-13T00:44:03.847 INFO:tasks.workunit.client.0.smithi077.stdout: write object-414 2024-09-13T00:44:03.847 INFO:tasks.workunit.client.0.smithi077.stdout: write object-445 2024-09-13T00:44:03.847 INFO:tasks.workunit.client.0.smithi077.stdout: write object-446 2024-09-13T00:44:03.847 INFO:tasks.workunit.client.0.smithi077.stdout: write object-447 2024-09-13T00:44:03.847 INFO:tasks.workunit.client.0.smithi077.stdout: write object-454 2024-09-13T00:44:03.847 INFO:tasks.workunit.client.0.smithi077.stdout: write object-459 2024-09-13T00:44:03.847 INFO:tasks.workunit.client.0.smithi077.stdout: write object-470 2024-09-13T00:44:03.847 INFO:tasks.workunit.client.0.smithi077.stdout: write object-487 2024-09-13T00:44:03.848 INFO:tasks.workunit.client.0.smithi077.stdout: write object-498 2024-09-13T00:44:03.848 INFO:tasks.workunit.client.0.smithi077.stdout: write object-507 2024-09-13T00:44:03.848 INFO:tasks.workunit.client.0.smithi077.stdout: write object-514 2024-09-13T00:44:03.848 INFO:tasks.workunit.client.0.smithi077.stdout: write object-516 2024-09-13T00:44:03.848 INFO:tasks.workunit.client.0.smithi077.stdout: write object-521 2024-09-13T00:44:03.848 INFO:tasks.workunit.client.0.smithi077.stdout: write object-536 2024-09-13T00:44:03.848 INFO:tasks.workunit.client.0.smithi077.stdout: write object-551 2024-09-13T00:44:03.848 INFO:tasks.workunit.client.0.smithi077.stdout: write object-561 2024-09-13T00:44:03.848 INFO:tasks.workunit.client.0.smithi077.stdout: write object-564 2024-09-13T00:44:03.848 INFO:tasks.workunit.client.0.smithi077.stdout: write object-594 2024-09-13T00:44:03.848 INFO:tasks.workunit.client.0.smithi077.stdout: write object-612 2024-09-13T00:44:03.848 INFO:tasks.workunit.client.0.smithi077.stdout: write object-625 2024-09-13T00:44:03.848 INFO:tasks.workunit.client.0.smithi077.stdout: write object-652 2024-09-13T00:44:03.849 INFO:tasks.workunit.client.0.smithi077.stdout: write object-681 2024-09-13T00:44:03.849 INFO:tasks.workunit.client.0.smithi077.stdout: write object-682 2024-09-13T00:44:03.849 INFO:tasks.workunit.client.0.smithi077.stdout: write object-687 2024-09-13T00:44:03.849 INFO:tasks.workunit.client.0.smithi077.stdout: write object-688 2024-09-13T00:44:03.849 INFO:tasks.workunit.client.0.smithi077.stdout: write object-695 2024-09-13T00:44:03.849 INFO:tasks.workunit.client.0.smithi077.stdout: write object-705 2024-09-13T00:44:03.849 INFO:tasks.workunit.client.0.smithi077.stdout: write object-718 2024-09-13T00:44:03.849 INFO:tasks.workunit.client.0.smithi077.stdout: write object-764 2024-09-13T00:44:03.849 INFO:tasks.workunit.client.0.smithi077.stdout: write object-767 2024-09-13T00:44:03.849 INFO:tasks.workunit.client.0.smithi077.stdout: write object-771 2024-09-13T00:44:03.849 INFO:tasks.workunit.client.0.smithi077.stdout: write object-778 2024-09-13T00:44:03.849 INFO:tasks.workunit.client.0.smithi077.stdout: write object-801 2024-09-13T00:44:03.849 INFO:tasks.workunit.client.0.smithi077.stdout: write object-807 2024-09-13T00:44:03.850 INFO:tasks.workunit.client.0.smithi077.stdout: write object-811 2024-09-13T00:44:03.850 INFO:tasks.workunit.client.0.smithi077.stdout: write object-814 2024-09-13T00:44:03.850 INFO:tasks.workunit.client.0.smithi077.stdout: write object-819 2024-09-13T00:44:03.850 INFO:tasks.workunit.client.0.smithi077.stdout: write object-829 2024-09-13T00:44:03.850 INFO:tasks.workunit.client.0.smithi077.stdout: write object-839 2024-09-13T00:44:03.850 INFO:tasks.workunit.client.0.smithi077.stdout: write object-855 2024-09-13T00:44:03.850 INFO:tasks.workunit.client.0.smithi077.stdout: write object-856 2024-09-13T00:44:03.850 INFO:tasks.workunit.client.0.smithi077.stdout: write object-886 2024-09-13T00:44:03.850 INFO:tasks.workunit.client.0.smithi077.stdout: write object-895 2024-09-13T00:44:03.850 INFO:tasks.workunit.client.0.smithi077.stdout: write object-934 2024-09-13T00:44:03.850 INFO:tasks.workunit.client.0.smithi077.stdout: write object-941 2024-09-13T00:44:04.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:04 smithi077 bash[83127]: cluster 2024-09-13T00:44:02.455224+0000 mgr.y (mgr.25304) 225 : cluster [DBG] pgmap v411: 105 pgs: 105 active+clean; 13 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 220 op/s; 2.2 KiB/s, 39 keys/s, 11 objects/s recovering 2024-09-13T00:44:04.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:04 smithi077 bash[83127]: cluster 2024-09-13T00:44:03.150290+0000 mon.a (mon.0) 668 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/468 objects degraded (1.068%), 2 pgs degraded) 2024-09-13T00:44:04.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:04 smithi077 bash[83127]: cluster 2024-09-13T00:44:03.165941+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e547: 8 total, 8 up, 8 in 2024-09-13T00:44:04.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:04 smithi077 bash[80681]: cluster 2024-09-13T00:44:02.455224+0000 mgr.y (mgr.25304) 225 : cluster [DBG] pgmap v411: 105 pgs: 105 active+clean; 13 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 220 op/s; 2.2 KiB/s, 39 keys/s, 11 objects/s recovering 2024-09-13T00:44:04.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:04 smithi077 bash[80681]: cluster 2024-09-13T00:44:03.150290+0000 mon.a (mon.0) 668 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/468 objects degraded (1.068%), 2 pgs degraded) 2024-09-13T00:44:04.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:04 smithi077 bash[80681]: cluster 2024-09-13T00:44:03.165941+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e547: 8 total, 8 up, 8 in 2024-09-13T00:44:04.529 INFO:tasks.workunit.client.0.smithi077.stdout: write object-948 2024-09-13T00:44:04.529 INFO:tasks.workunit.client.0.smithi077.stdout: write object-951 2024-09-13T00:44:04.530 INFO:tasks.workunit.client.0.smithi077.stdout: write object-957 2024-09-13T00:44:04.530 INFO:tasks.workunit.client.0.smithi077.stdout: write object-968 2024-09-13T00:44:04.530 INFO:tasks.workunit.client.0.smithi077.stdout: write object-981 2024-09-13T00:44:04.530 INFO:tasks.workunit.client.0.smithi077.stdout: write object-982 2024-09-13T00:44:04.530 INFO:tasks.workunit.client.0.smithi077.stdout: write object-993 2024-09-13T00:44:04.530 INFO:tasks.workunit.client.0.smithi077.stdout: write object-1017 2024-09-13T00:44:04.530 INFO:tasks.workunit.client.0.smithi077.stdout:verify written data by reading 2024-09-13T00:44:04.530 INFO:tasks.workunit.client.0.smithi077.stdout: read object-1 2024-09-13T00:44:04.530 INFO:tasks.workunit.client.0.smithi077.stdout:made snapshot "image383@parent_snap" and protect it 2024-09-13T00:44:04.531 INFO:tasks.workunit.client.0.smithi077.stdout:made and opened clone "image384" 2024-09-13T00:44:04.531 INFO:tasks.workunit.client.0.smithi077.stdout:flattening clone: "image384" 2024-09-13T00:44:04.531 INFO:tasks.workunit.client.0.smithi077.stdout:check whether child image has the same set of objects as parent 2024-09-13T00:44:04.531 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.FlattenNoEmptyObjects (1289 ms) 2024-09-13T00:44:04.531 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.PoolMetadataConfApply 2024-09-13T00:44:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:04 smithi088 bash[68960]: cluster 2024-09-13T00:44:02.455224+0000 mgr.y (mgr.25304) 225 : cluster [DBG] pgmap v411: 105 pgs: 105 active+clean; 13 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.0 MiB/s wr, 220 op/s; 2.2 KiB/s, 39 keys/s, 11 objects/s recovering 2024-09-13T00:44:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:04 smithi088 bash[68960]: cluster 2024-09-13T00:44:03.150290+0000 mon.a (mon.0) 668 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/468 objects degraded (1.068%), 2 pgs degraded) 2024-09-13T00:44:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:04 smithi088 bash[68960]: cluster 2024-09-13T00:44:03.165941+0000 mon.a (mon.0) 669 : cluster [DBG] osdmap e547: 8 total, 8 up, 8 in 2024-09-13T00:44:04.649 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.PoolMetadataConfApply (121 ms) 2024-09-13T00:44:04.649 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.Sparsify 2024-09-13T00:44:04.695 INFO:tasks.workunit.client.0.smithi077.stdout:sparsify_supported=1 2024-09-13T00:44:04.695 INFO:tasks.workunit.client.0.smithi077.stdout:sparse_read_supported=1 2024-09-13T00:44:04.736 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.Sparsify (87 ms) 2024-09-13T00:44:04.736 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.SparsifyClone 2024-09-13T00:44:04.773 INFO:tasks.workunit.client.0.smithi077.stdout:sparsify_supported=1 2024-09-13T00:44:05.028 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:44:04 smithi077 bash[73137]: [13/Sep/2024:00:44:04] ENGINE Bus STOPPING 2024-09-13T00:44:05.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:05 smithi077 bash[80681]: cluster 2024-09-13T00:44:04.178468+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2024-09-13T00:44:05.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:05 smithi077 bash[80681]: audit 2024-09-13T00:44:04.564182+0000 mon.a (mon.0) 671 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:44:05.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:05 smithi077 bash[83127]: cluster 2024-09-13T00:44:04.178468+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2024-09-13T00:44:05.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:05 smithi077 bash[83127]: audit 2024-09-13T00:44:04.564182+0000 mon.a (mon.0) 671 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:44:05.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:44:05 smithi077 bash[73137]: [13/Sep/2024:00:44:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:44:05.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:44:05 smithi077 bash[73137]: [13/Sep/2024:00:44:05] ENGINE Bus STOPPED 2024-09-13T00:44:05.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:44:05 smithi077 bash[73137]: [13/Sep/2024:00:44:05] ENGINE Bus STARTING 2024-09-13T00:44:05.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:44:05 smithi077 bash[73137]: [13/Sep/2024:00:44:05] ENGINE Serving on http://:::9283 2024-09-13T00:44:05.529 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:44:05 smithi077 bash[73137]: [13/Sep/2024:00:44:05] ENGINE Bus STARTED 2024-09-13T00:44:05.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:05 smithi088 bash[68960]: cluster 2024-09-13T00:44:04.178468+0000 mon.a (mon.0) 670 : cluster [DBG] osdmap e548: 8 total, 8 up, 8 in 2024-09-13T00:44:05.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:05 smithi088 bash[68960]: audit 2024-09-13T00:44:04.564182+0000 mon.a (mon.0) 671 : audit [INF] from='client.? ' entity='client.admin' 2024-09-13T00:44:06.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:06 smithi077 bash[80681]: cluster 2024-09-13T00:44:04.456168+0000 mgr.y (mgr.25304) 226 : cluster [DBG] pgmap v414: 105 pgs: 105 active+clean; 4.8 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 81 KiB/s wr, 393 op/s; 3 B/s, 6 keys/s, 1 objects/s recovering 2024-09-13T00:44:06.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:06 smithi077 bash[80681]: cluster 2024-09-13T00:44:05.569932+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2024-09-13T00:44:06.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:06 smithi077 bash[80681]: cluster 2024-09-13T00:44:05.923549+0000 mon.a (mon.0) 673 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2024-09-13T00:44:06.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:06 smithi077 bash[83127]: cluster 2024-09-13T00:44:04.456168+0000 mgr.y (mgr.25304) 226 : cluster [DBG] pgmap v414: 105 pgs: 105 active+clean; 4.8 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 81 KiB/s wr, 393 op/s; 3 B/s, 6 keys/s, 1 objects/s recovering 2024-09-13T00:44:06.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:06 smithi077 bash[83127]: cluster 2024-09-13T00:44:05.569932+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2024-09-13T00:44:06.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:06 smithi077 bash[83127]: cluster 2024-09-13T00:44:05.923549+0000 mon.a (mon.0) 673 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2024-09-13T00:44:06.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:06 smithi088 bash[68960]: cluster 2024-09-13T00:44:04.456168+0000 mgr.y (mgr.25304) 226 : cluster [DBG] pgmap v414: 105 pgs: 105 active+clean; 4.8 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 249 KiB/s rd, 81 KiB/s wr, 393 op/s; 3 B/s, 6 keys/s, 1 objects/s recovering 2024-09-13T00:44:06.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:06 smithi088 bash[68960]: cluster 2024-09-13T00:44:05.569932+0000 mon.a (mon.0) 672 : cluster [DBG] osdmap e549: 8 total, 8 up, 8 in 2024-09-13T00:44:06.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:06 smithi088 bash[68960]: cluster 2024-09-13T00:44:05.923549+0000 mon.a (mon.0) 673 : cluster [DBG] osdmap e550: 8 total, 8 up, 8 in 2024-09-13T00:44:06.946 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.SparsifyClone (2210 ms) 2024-09-13T00:44:06.946 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestInternal.MissingDataPool 2024-09-13T00:44:08.147 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestInternal.MissingDataPool (1201 ms) 2024-09-13T00:44:08.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:07 smithi077 bash[80681]: cluster 2024-09-13T00:44:06.456993+0000 mgr.y (mgr.25304) 227 : cluster [DBG] pgmap v417: 105 pgs: 105 active+clean; 4.9 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 329 KiB/s rd, 103 KiB/s wr, 504 op/s 2024-09-13T00:44:08.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:07 smithi077 bash[80681]: cluster 2024-09-13T00:44:06.924683+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2024-09-13T00:44:08.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:07 smithi077 bash[80681]: audit 2024-09-13T00:44:07.690665+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:08.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:07 smithi077 bash[80681]: audit 2024-09-13T00:44:07.699645+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:08.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:07 smithi077 bash[80681]: audit 2024-09-13T00:44:07.707302+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:08.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:07 smithi077 bash[83127]: cluster 2024-09-13T00:44:06.456993+0000 mgr.y (mgr.25304) 227 : cluster [DBG] pgmap v417: 105 pgs: 105 active+clean; 4.9 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 329 KiB/s rd, 103 KiB/s wr, 504 op/s 2024-09-13T00:44:08.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:07 smithi077 bash[83127]: cluster 2024-09-13T00:44:06.924683+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2024-09-13T00:44:08.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:07 smithi077 bash[83127]: audit 2024-09-13T00:44:07.690665+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:08.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:07 smithi077 bash[83127]: audit 2024-09-13T00:44:07.699645+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:08.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:07 smithi077 bash[83127]: audit 2024-09-13T00:44:07.707302+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:08.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:07 smithi088 bash[68960]: cluster 2024-09-13T00:44:06.456993+0000 mgr.y (mgr.25304) 227 : cluster [DBG] pgmap v417: 105 pgs: 105 active+clean; 4.9 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 329 KiB/s rd, 103 KiB/s wr, 504 op/s 2024-09-13T00:44:08.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:07 smithi088 bash[68960]: cluster 2024-09-13T00:44:06.924683+0000 mon.a (mon.0) 674 : cluster [DBG] osdmap e551: 8 total, 8 up, 8 in 2024-09-13T00:44:08.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:07 smithi088 bash[68960]: audit 2024-09-13T00:44:07.690665+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:08.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:07 smithi088 bash[68960]: audit 2024-09-13T00:44:07.699645+0000 mon.a (mon.0) 676 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:08.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:07 smithi088 bash[68960]: audit 2024-09-13T00:44:07.707302+0000 mon.a (mon.0) 677 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:08.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:44:08] "GET /metrics HTTP/1.1" 200 36410 "" "Prometheus/2.43.0" 2024-09-13T00:44:08.947 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 34 tests from TestInternal (38093 ms total) 2024-09-13T00:44:08.947 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:44:08.948 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 17 tests from TestImageWatcher 2024-09-13T00:44:09.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[83127]: cluster 2024-09-13T00:44:07.935265+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2024-09-13T00:44:09.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[83127]: audit 2024-09-13T00:44:08.206654+0000 mon.a (mon.0) 679 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:44:09.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[83127]: audit 2024-09-13T00:44:08.208470+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:44:09.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[83127]: audit 2024-09-13T00:44:08.216400+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:09.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[83127]: audit 2024-09-13T00:44:08.351585+0000 mon.a (mon.0) 682 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:44:09.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[83127]: audit 2024-09-13T00:44:08.354233+0000 mon.a (mon.0) 683 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:44:09.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[83127]: audit 2024-09-13T00:44:08.355379+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:44:09.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[83127]: audit 2024-09-13T00:44:08.366140+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:09.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[80681]: cluster 2024-09-13T00:44:07.935265+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2024-09-13T00:44:09.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[80681]: audit 2024-09-13T00:44:08.206654+0000 mon.a (mon.0) 679 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:44:09.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[80681]: audit 2024-09-13T00:44:08.208470+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:44:09.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[80681]: audit 2024-09-13T00:44:08.216400+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:09.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[80681]: audit 2024-09-13T00:44:08.351585+0000 mon.a (mon.0) 682 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:44:09.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[80681]: audit 2024-09-13T00:44:08.354233+0000 mon.a (mon.0) 683 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:44:09.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[80681]: audit 2024-09-13T00:44:08.355379+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:44:09.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:08 smithi077 bash[80681]: audit 2024-09-13T00:44:08.366140+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:09.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:08 smithi088 bash[68960]: cluster 2024-09-13T00:44:07.935265+0000 mon.a (mon.0) 678 : cluster [DBG] osdmap e552: 8 total, 8 up, 8 in 2024-09-13T00:44:09.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:08 smithi088 bash[68960]: audit 2024-09-13T00:44:08.206654+0000 mon.a (mon.0) 679 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:44:09.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:08 smithi088 bash[68960]: audit 2024-09-13T00:44:08.208470+0000 mon.a (mon.0) 680 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:44:09.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:08 smithi088 bash[68960]: audit 2024-09-13T00:44:08.216400+0000 mon.a (mon.0) 681 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:09.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:08 smithi088 bash[68960]: audit 2024-09-13T00:44:08.351585+0000 mon.a (mon.0) 682 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:44:09.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:08 smithi088 bash[68960]: audit 2024-09-13T00:44:08.354233+0000 mon.a (mon.0) 683 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:44:09.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:08 smithi088 bash[68960]: audit 2024-09-13T00:44:08.355379+0000 mon.a (mon.0) 684 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:44:09.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:08 smithi088 bash[68960]: audit 2024-09-13T00:44:08.366140+0000 mon.a (mon.0) 685 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:10.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:09 smithi077 bash[83127]: cluster 2024-09-13T00:44:08.457665+0000 mgr.y (mgr.25304) 228 : cluster [DBG] pgmap v420: 105 pgs: 105 active+clean; 4.9 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 25 KiB/s wr, 130 op/s 2024-09-13T00:44:10.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:09 smithi077 bash[83127]: cluster 2024-09-13T00:44:08.939664+0000 mon.a (mon.0) 686 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2024-09-13T00:44:10.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:09 smithi077 bash[80681]: cluster 2024-09-13T00:44:08.457665+0000 mgr.y (mgr.25304) 228 : cluster [DBG] pgmap v420: 105 pgs: 105 active+clean; 4.9 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 25 KiB/s wr, 130 op/s 2024-09-13T00:44:10.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:09 smithi077 bash[80681]: cluster 2024-09-13T00:44:08.939664+0000 mon.a (mon.0) 686 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2024-09-13T00:44:10.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:09 smithi088 bash[68960]: cluster 2024-09-13T00:44:08.457665+0000 mgr.y (mgr.25304) 228 : cluster [DBG] pgmap v420: 105 pgs: 105 active+clean; 4.9 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 89 KiB/s rd, 25 KiB/s wr, 130 op/s 2024-09-13T00:44:10.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:09 smithi088 bash[68960]: cluster 2024-09-13T00:44:08.939664+0000 mon.a (mon.0) 686 : cluster [DBG] osdmap e553: 8 total, 8 up, 8 in 2024-09-13T00:44:10.969 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifyHeaderUpdate 2024-09-13T00:44:11.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:11 smithi077 bash[80681]: cluster 2024-09-13T00:44:09.957292+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2024-09-13T00:44:11.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:11 smithi077 bash[80681]: audit 2024-09-13T00:44:09.962008+0000 mon.c (mon.2) 22 : audit [INF] from='client.? 172.21.15.77:0/1911795692' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:11.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:11 smithi077 bash[80681]: audit 2024-09-13T00:44:09.970747+0000 mon.a (mon.0) 688 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:11.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:11 smithi077 bash[80681]: audit 2024-09-13T00:44:10.958567+0000 mon.a (mon.0) 689 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:11.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:11 smithi077 bash[80681]: cluster 2024-09-13T00:44:10.959490+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2024-09-13T00:44:11.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:11 smithi077 bash[83127]: cluster 2024-09-13T00:44:09.957292+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2024-09-13T00:44:11.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:11 smithi077 bash[83127]: audit 2024-09-13T00:44:09.962008+0000 mon.c (mon.2) 22 : audit [INF] from='client.? 172.21.15.77:0/1911795692' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:11.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:11 smithi077 bash[83127]: audit 2024-09-13T00:44:09.970747+0000 mon.a (mon.0) 688 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:11.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:11 smithi077 bash[83127]: audit 2024-09-13T00:44:10.958567+0000 mon.a (mon.0) 689 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:11.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:11 smithi077 bash[83127]: cluster 2024-09-13T00:44:10.959490+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2024-09-13T00:44:11.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:11 smithi088 bash[68960]: cluster 2024-09-13T00:44:09.957292+0000 mon.a (mon.0) 687 : cluster [DBG] osdmap e554: 8 total, 8 up, 8 in 2024-09-13T00:44:11.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:11 smithi088 bash[68960]: audit 2024-09-13T00:44:09.962008+0000 mon.c (mon.2) 22 : audit [INF] from='client.? 172.21.15.77:0/1911795692' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:11.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:11 smithi088 bash[68960]: audit 2024-09-13T00:44:09.970747+0000 mon.a (mon.0) 688 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:11.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:11 smithi088 bash[68960]: audit 2024-09-13T00:44:10.958567+0000 mon.a (mon.0) 689 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:11.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:11 smithi088 bash[68960]: cluster 2024-09-13T00:44:10.959490+0000 mon.a (mon.0) 690 : cluster [DBG] osdmap e555: 8 total, 8 up, 8 in 2024-09-13T00:44:12.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:12 smithi077 bash[80681]: cluster 2024-09-13T00:44:10.458359+0000 mgr.y (mgr.25304) 229 : cluster [DBG] pgmap v423: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:12.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:12 smithi077 bash[80681]: cluster 2024-09-13T00:44:10.971616+0000 mon.a (mon.0) 691 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:44:12.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:12 smithi077 bash[80681]: cluster 2024-09-13T00:44:11.963632+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2024-09-13T00:44:12.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:12 smithi077 bash[83127]: cluster 2024-09-13T00:44:10.458359+0000 mgr.y (mgr.25304) 229 : cluster [DBG] pgmap v423: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:12.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:12 smithi077 bash[83127]: cluster 2024-09-13T00:44:10.971616+0000 mon.a (mon.0) 691 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:44:12.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:12 smithi077 bash[83127]: cluster 2024-09-13T00:44:11.963632+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2024-09-13T00:44:12.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:12 smithi088 bash[68960]: cluster 2024-09-13T00:44:10.458359+0000 mgr.y (mgr.25304) 229 : cluster [DBG] pgmap v423: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:12.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:12 smithi088 bash[68960]: cluster 2024-09-13T00:44:10.971616+0000 mon.a (mon.0) 691 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:44:12.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:12 smithi088 bash[68960]: cluster 2024-09-13T00:44:11.963632+0000 mon.a (mon.0) 692 : cluster [DBG] osdmap e556: 8 total, 8 up, 8 in 2024-09-13T00:44:12.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:44:12 smithi088 bash[61791]: ts=2024-09-13T00:44:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:44:13.020 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifyHeaderUpdate (2051 ms) 2024-09-13T00:44:13.020 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifyFlatten 2024-09-13T00:44:13.064 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifyFlatten (45 ms) 2024-09-13T00:44:13.064 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifyResize 2024-09-13T00:44:13.128 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifyResize (64 ms) 2024-09-13T00:44:13.129 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifyRebuildObjectMap 2024-09-13T00:44:13.173 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifyRebuildObjectMap (44 ms) 2024-09-13T00:44:13.173 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifyUpdateFeatures 2024-09-13T00:44:13.217 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifyUpdateFeatures (45 ms) 2024-09-13T00:44:13.217 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifySnapCreate 2024-09-13T00:44:13.261 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifySnapCreate (43 ms) 2024-09-13T00:44:13.349 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifySnapCreateError 2024-09-13T00:44:13.349 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifySnapCreateError (47 ms) 2024-09-13T00:44:13.349 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifySnapRename 2024-09-13T00:44:13.365 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifySnapRename (57 ms) 2024-09-13T00:44:13.365 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifySnapRenameError 2024-09-13T00:44:13.416 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifySnapRenameError (51 ms) 2024-09-13T00:44:13.416 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifySnapRemove 2024-09-13T00:44:13.468 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifySnapRemove (52 ms) 2024-09-13T00:44:13.468 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifySnapProtect 2024-09-13T00:44:13.512 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifySnapProtect (44 ms) 2024-09-13T00:44:13.512 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifySnapUnprotect 2024-09-13T00:44:13.561 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifySnapUnprotect (49 ms) 2024-09-13T00:44:13.561 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifyRename 2024-09-13T00:44:13.597 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifyRename (37 ms) 2024-09-13T00:44:13.597 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifyAsyncTimedOut 2024-09-13T00:44:13.633 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifyAsyncTimedOut (35 ms) 2024-09-13T00:44:13.633 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifyAsyncError 2024-09-13T00:44:13.675 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifyAsyncError (43 ms) 2024-09-13T00:44:13.675 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifyAsyncCompleteError 2024-09-13T00:44:13.716 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifyAsyncCompleteError (40 ms) 2024-09-13T00:44:13.716 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestImageWatcher.NotifyAsyncRequestTimedOut 2024-09-13T00:44:13.764 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestImageWatcher.NotifyAsyncRequestTimedOut (48 ms) 2024-09-13T00:44:14.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:13 smithi077 bash[80681]: cluster 2024-09-13T00:44:12.459000+0000 mgr.y (mgr.25304) 230 : cluster [DBG] pgmap v426: 105 pgs: 26 unknown, 79 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:14.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:13 smithi077 bash[80681]: cluster 2024-09-13T00:44:12.968181+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2024-09-13T00:44:14.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:13 smithi077 bash[80681]: cluster 2024-09-13T00:44:13.001431+0000 mon.a (mon.0) 694 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:44:14.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:13 smithi077 bash[80681]: audit 2024-09-13T00:44:13.421014+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:14.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:13 smithi077 bash[80681]: audit 2024-09-13T00:44:13.422405+0000 mon.a (mon.0) 696 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:14.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:13 smithi077 bash[83127]: cluster 2024-09-13T00:44:12.459000+0000 mgr.y (mgr.25304) 230 : cluster [DBG] pgmap v426: 105 pgs: 26 unknown, 79 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:14.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:13 smithi077 bash[83127]: cluster 2024-09-13T00:44:12.968181+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2024-09-13T00:44:14.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:13 smithi077 bash[83127]: cluster 2024-09-13T00:44:13.001431+0000 mon.a (mon.0) 694 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:44:14.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:13 smithi077 bash[83127]: audit 2024-09-13T00:44:13.421014+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:14.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:13 smithi077 bash[83127]: audit 2024-09-13T00:44:13.422405+0000 mon.a (mon.0) 696 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:14.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:13 smithi088 bash[68960]: cluster 2024-09-13T00:44:12.459000+0000 mgr.y (mgr.25304) 230 : cluster [DBG] pgmap v426: 105 pgs: 26 unknown, 79 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:14.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:13 smithi088 bash[68960]: cluster 2024-09-13T00:44:12.968181+0000 mon.a (mon.0) 693 : cluster [DBG] osdmap e557: 8 total, 8 up, 8 in 2024-09-13T00:44:14.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:13 smithi088 bash[68960]: cluster 2024-09-13T00:44:13.001431+0000 mon.a (mon.0) 694 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:44:14.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:13 smithi088 bash[68960]: audit 2024-09-13T00:44:13.421014+0000 mon.a (mon.0) 695 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:14.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:13 smithi088 bash[68960]: audit 2024-09-13T00:44:13.422405+0000 mon.a (mon.0) 696 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:14.436 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 17 tests from TestImageWatcher (2795 ms total) 2024-09-13T00:44:14.436 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:44:14.437 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 160 tests from TestLibRBD 2024-09-13T00:44:14.486 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.CreateAndStat 2024-09-13T00:44:15.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:15 smithi077 bash[80681]: cluster 2024-09-13T00:44:14.429421+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e558: 8 total, 8 up, 8 in 2024-09-13T00:44:15.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:15 smithi077 bash[83127]: cluster 2024-09-13T00:44:14.429421+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e558: 8 total, 8 up, 8 in 2024-09-13T00:44:15.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:15 smithi088 bash[68960]: cluster 2024-09-13T00:44:14.429421+0000 mon.a (mon.0) 697 : cluster [DBG] osdmap e558: 8 total, 8 up, 8 in 2024-09-13T00:44:16.459 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:16.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:16 smithi077 bash[80681]: cluster 2024-09-13T00:44:14.459666+0000 mgr.y (mgr.25304) 231 : cluster [DBG] pgmap v429: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:16.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:16 smithi077 bash[80681]: cluster 2024-09-13T00:44:15.446316+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e559: 8 total, 8 up, 8 in 2024-09-13T00:44:16.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:16 smithi077 bash[80681]: audit 2024-09-13T00:44:15.473464+0000 mon.a (mon.0) 699 : audit [INF] from='client.? 172.21.15.77:0/2403490909' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:16.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:16 smithi077 bash[83127]: cluster 2024-09-13T00:44:14.459666+0000 mgr.y (mgr.25304) 231 : cluster [DBG] pgmap v429: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:16.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:16 smithi077 bash[83127]: cluster 2024-09-13T00:44:15.446316+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e559: 8 total, 8 up, 8 in 2024-09-13T00:44:16.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:16 smithi077 bash[83127]: audit 2024-09-13T00:44:15.473464+0000 mon.a (mon.0) 699 : audit [INF] from='client.? 172.21.15.77:0/2403490909' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:16.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:16 smithi088 bash[68960]: cluster 2024-09-13T00:44:14.459666+0000 mgr.y (mgr.25304) 231 : cluster [DBG] pgmap v429: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:16.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:16 smithi088 bash[68960]: cluster 2024-09-13T00:44:15.446316+0000 mon.a (mon.0) 698 : cluster [DBG] osdmap e559: 8 total, 8 up, 8 in 2024-09-13T00:44:16.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:16 smithi088 bash[68960]: audit 2024-09-13T00:44:15.473464+0000 mon.a (mon.0) 699 : audit [INF] from='client.? 172.21.15.77:0/2403490909' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:17.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:17 smithi077 bash[80681]: audit 2024-09-13T00:44:16.447274+0000 mon.a (mon.0) 700 : audit [INF] from='client.? 172.21.15.77:0/2403490909' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:17.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:17 smithi077 bash[80681]: cluster 2024-09-13T00:44:16.447532+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e560: 8 total, 8 up, 8 in 2024-09-13T00:44:17.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:17 smithi077 bash[80681]: cluster 2024-09-13T00:44:16.460356+0000 mgr.y (mgr.25304) 232 : cluster [DBG] pgmap v432: 105 pgs: 5 creating+peering, 27 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:17.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:17 smithi077 bash[83127]: audit 2024-09-13T00:44:16.447274+0000 mon.a (mon.0) 700 : audit [INF] from='client.? 172.21.15.77:0/2403490909' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:17.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:17 smithi077 bash[83127]: cluster 2024-09-13T00:44:16.447532+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e560: 8 total, 8 up, 8 in 2024-09-13T00:44:17.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:17 smithi077 bash[83127]: cluster 2024-09-13T00:44:16.460356+0000 mgr.y (mgr.25304) 232 : cluster [DBG] pgmap v432: 105 pgs: 5 creating+peering, 27 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:17.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:17 smithi088 bash[68960]: audit 2024-09-13T00:44:16.447274+0000 mon.a (mon.0) 700 : audit [INF] from='client.? 172.21.15.77:0/2403490909' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:17.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:17 smithi088 bash[68960]: cluster 2024-09-13T00:44:16.447532+0000 mon.a (mon.0) 701 : cluster [DBG] osdmap e560: 8 total, 8 up, 8 in 2024-09-13T00:44:17.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:17 smithi088 bash[68960]: cluster 2024-09-13T00:44:16.460356+0000 mgr.y (mgr.25304) 232 : cluster [DBG] pgmap v432: 105 pgs: 5 creating+peering, 27 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:18.536 INFO:tasks.workunit.client.0.smithi077.stdout:image has size 2097152 and order 22 2024-09-13T00:44:18.536 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.CreateAndStat (4049 ms) 2024-09-13T00:44:18.536 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.CreateWithSameDataPool 2024-09-13T00:44:18.536 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:18.572 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.CreateWithSameDataPool (36 ms) 2024-09-13T00:44:18.572 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.CreateAndStatPP 2024-09-13T00:44:18.573 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:18.623 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.CreateAndStatPP (51 ms) 2024-09-13T00:44:18.623 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.GetId 2024-09-13T00:44:18.623 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:18.669 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.GetId (44 ms) 2024-09-13T00:44:18.669 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.GetIdPP 2024-09-13T00:44:18.669 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:18.712 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.GetIdPP (46 ms) 2024-09-13T00:44:18.712 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefix 2024-09-13T00:44:18.713 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:18.745 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.GetBlockNamePrefix (33 ms) 2024-09-13T00:44:18.746 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefixPP 2024-09-13T00:44:18.746 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:18.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:44:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:44:18] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2024-09-13T00:44:18.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:18 smithi077 bash[83127]: cluster 2024-09-13T00:44:17.474413+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e561: 8 total, 8 up, 8 in 2024-09-13T00:44:18.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:18 smithi077 bash[80681]: cluster 2024-09-13T00:44:17.474413+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e561: 8 total, 8 up, 8 in 2024-09-13T00:44:18.779 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.GetBlockNamePrefixPP (34 ms) 2024-09-13T00:44:18.779 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestGetCreateTimestamp 2024-09-13T00:44:18.779 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:18.810 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestGetCreateTimestamp (31 ms) 2024-09-13T00:44:18.810 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.GetCreateTimestampPP 2024-09-13T00:44:18.810 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:18.841 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.GetCreateTimestampPP (30 ms) 2024-09-13T00:44:18.841 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.OpenAio 2024-09-13T00:44:18.841 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:18.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:18 smithi088 bash[68960]: cluster 2024-09-13T00:44:17.474413+0000 mon.a (mon.0) 702 : cluster [DBG] osdmap e561: 8 total, 8 up, 8 in 2024-09-13T00:44:18.874 INFO:tasks.workunit.client.0.smithi077.stdout:image has size 2097152 and order 22 2024-09-13T00:44:18.875 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.OpenAio (33 ms) 2024-09-13T00:44:18.875 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.OpenAioFail 2024-09-13T00:44:18.879 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.OpenAioFail (5 ms) 2024-09-13T00:44:18.880 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.OpenAioPP 2024-09-13T00:44:18.880 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:18.932 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.OpenAioPP (52 ms) 2024-09-13T00:44:18.932 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.OpenAioFailPP 2024-09-13T00:44:18.939 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.OpenAioFailPP (8 ms) 2024-09-13T00:44:18.939 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ResizeAndStat 2024-09-13T00:44:18.939 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:19.032 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ResizeAndStat (92 ms) 2024-09-13T00:44:19.032 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ResizeAndStatPP 2024-09-13T00:44:19.032 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:19.115 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ResizeAndStatPP (83 ms) 2024-09-13T00:44:19.115 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResize 2024-09-13T00:44:19.115 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:19.198 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.UpdateWatchAndResize (83 ms) 2024-09-13T00:44:19.198 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResizePP 2024-09-13T00:44:19.198 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:19.284 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.UpdateWatchAndResizePP (87 ms) 2024-09-13T00:44:19.285 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCreateLsDelete 2024-09-13T00:44:20.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:19 smithi077 bash[80681]: cluster 2024-09-13T00:44:18.485147+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e562: 8 total, 8 up, 8 in 2024-09-13T00:44:20.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:19 smithi077 bash[83127]: cluster 2024-09-13T00:44:18.485147+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e562: 8 total, 8 up, 8 in 2024-09-13T00:44:20.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:19 smithi088 bash[68960]: cluster 2024-09-13T00:44:18.485147+0000 mon.a (mon.0) 703 : cluster [DBG] osdmap e562: 8 total, 8 up, 8 in 2024-09-13T00:44:20.862 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:21.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:20 smithi077 bash[80681]: cluster 2024-09-13T00:44:18.461000+0000 mgr.y (mgr.25304) 233 : cluster [DBG] pgmap v434: 105 pgs: 5 creating+peering, 27 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:21.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:20 smithi077 bash[80681]: cluster 2024-09-13T00:44:19.814651+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e563: 8 total, 8 up, 8 in 2024-09-13T00:44:21.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:20 smithi077 bash[80681]: audit 2024-09-13T00:44:19.819676+0000 mon.c (mon.2) 23 : audit [INF] from='client.? 172.21.15.77:0/738055495' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:21.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:20 smithi077 bash[80681]: audit 2024-09-13T00:44:19.820467+0000 mon.a (mon.0) 705 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:21.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:20 smithi077 bash[83127]: cluster 2024-09-13T00:44:18.461000+0000 mgr.y (mgr.25304) 233 : cluster [DBG] pgmap v434: 105 pgs: 5 creating+peering, 27 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:21.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:20 smithi077 bash[83127]: cluster 2024-09-13T00:44:19.814651+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e563: 8 total, 8 up, 8 in 2024-09-13T00:44:21.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:20 smithi077 bash[83127]: audit 2024-09-13T00:44:19.819676+0000 mon.c (mon.2) 23 : audit [INF] from='client.? 172.21.15.77:0/738055495' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:21.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:20 smithi077 bash[83127]: audit 2024-09-13T00:44:19.820467+0000 mon.a (mon.0) 705 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:21.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:20 smithi088 bash[68960]: cluster 2024-09-13T00:44:18.461000+0000 mgr.y (mgr.25304) 233 : cluster [DBG] pgmap v434: 105 pgs: 5 creating+peering, 27 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:21.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:20 smithi088 bash[68960]: cluster 2024-09-13T00:44:19.814651+0000 mon.a (mon.0) 704 : cluster [DBG] osdmap e563: 8 total, 8 up, 8 in 2024-09-13T00:44:21.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:20 smithi088 bash[68960]: audit 2024-09-13T00:44:19.819676+0000 mon.c (mon.2) 23 : audit [INF] from='client.? 172.21.15.77:0/738055495' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:21.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:20 smithi088 bash[68960]: audit 2024-09-13T00:44:19.820467+0000 mon.a (mon.0) 705 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:22.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:21 smithi077 bash[80681]: cluster 2024-09-13T00:44:20.461769+0000 mgr.y (mgr.25304) 234 : cluster [DBG] pgmap v437: 137 pgs: 32 unknown, 5 creating+peering, 100 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 17 KiB/s wr, 230 op/s 2024-09-13T00:44:22.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:21 smithi077 bash[80681]: cluster 2024-09-13T00:44:20.809007+0000 mon.a (mon.0) 706 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:44:22.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:21 smithi077 bash[80681]: audit 2024-09-13T00:44:20.829110+0000 mon.a (mon.0) 707 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:22.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:21 smithi077 bash[80681]: cluster 2024-09-13T00:44:20.829457+0000 mon.a (mon.0) 708 : cluster [DBG] osdmap e564: 8 total, 8 up, 8 in 2024-09-13T00:44:22.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:21 smithi077 bash[83127]: cluster 2024-09-13T00:44:20.461769+0000 mgr.y (mgr.25304) 234 : cluster [DBG] pgmap v437: 137 pgs: 32 unknown, 5 creating+peering, 100 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 17 KiB/s wr, 230 op/s 2024-09-13T00:44:22.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:21 smithi077 bash[83127]: cluster 2024-09-13T00:44:20.809007+0000 mon.a (mon.0) 706 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:44:22.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:21 smithi077 bash[83127]: audit 2024-09-13T00:44:20.829110+0000 mon.a (mon.0) 707 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:22.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:21 smithi077 bash[83127]: cluster 2024-09-13T00:44:20.829457+0000 mon.a (mon.0) 708 : cluster [DBG] osdmap e564: 8 total, 8 up, 8 in 2024-09-13T00:44:22.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:21 smithi088 bash[68960]: cluster 2024-09-13T00:44:20.461769+0000 mgr.y (mgr.25304) 234 : cluster [DBG] pgmap v437: 137 pgs: 32 unknown, 5 creating+peering, 100 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 172 KiB/s rd, 17 KiB/s wr, 230 op/s 2024-09-13T00:44:22.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:21 smithi088 bash[68960]: cluster 2024-09-13T00:44:20.809007+0000 mon.a (mon.0) 706 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:44:22.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:21 smithi088 bash[68960]: audit 2024-09-13T00:44:20.829110+0000 mon.a (mon.0) 707 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:22.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:21 smithi088 bash[68960]: cluster 2024-09-13T00:44:20.829457+0000 mon.a (mon.0) 708 : cluster [DBG] osdmap e564: 8 total, 8 up, 8 in 2024-09-13T00:44:22.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:44:22 smithi088 bash[61791]: ts=2024-09-13T00:44:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:44:22.885 INFO:tasks.workunit.client.0.smithi077.stdout:image: image18 2024-09-13T00:44:22.885 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image18 2024-09-13T00:44:22.885 INFO:tasks.workunit.client.0.smithi077.stdout:found image18 2024-09-13T00:44:22.885 INFO:tasks.workunit.client.0.smithi077.stdout:erased image18 2024-09-13T00:44:22.885 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:23.084 INFO:tasks.workunit.client.0.smithi077.stdout:image: image18 2024-09-13T00:44:23.084 INFO:tasks.workunit.client.0.smithi077.stdout:image: image19 2024-09-13T00:44:23.084 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image18 2024-09-13T00:44:23.084 INFO:tasks.workunit.client.0.smithi077.stdout:found image18 2024-09-13T00:44:23.084 INFO:tasks.workunit.client.0.smithi077.stdout:erased image18 2024-09-13T00:44:23.084 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image19 2024-09-13T00:44:23.084 INFO:tasks.workunit.client.0.smithi077.stdout:found image19 2024-09-13T00:44:23.085 INFO:tasks.workunit.client.0.smithi077.stdout:erased image19 2024-09-13T00:44:23.085 INFO:tasks.workunit.client.0.smithi077.stdout:image: image19 2024-09-13T00:44:23.085 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image19 2024-09-13T00:44:23.085 INFO:tasks.workunit.client.0.smithi077.stdout:found image19 2024-09-13T00:44:23.085 INFO:tasks.workunit.client.0.smithi077.stdout:erased image19 2024-09-13T00:44:23.085 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCreateLsDelete (3799 ms) 2024-09-13T00:44:23.085 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCreateLsDeletePP 2024-09-13T00:44:23.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:22 smithi088 bash[68960]: cluster 2024-09-13T00:44:21.832327+0000 mon.a (mon.0) 709 : cluster [DBG] osdmap e565: 8 total, 8 up, 8 in 2024-09-13T00:44:23.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:22 smithi077 bash[80681]: cluster 2024-09-13T00:44:21.832327+0000 mon.a (mon.0) 709 : cluster [DBG] osdmap e565: 8 total, 8 up, 8 in 2024-09-13T00:44:23.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:22 smithi077 bash[83127]: cluster 2024-09-13T00:44:21.832327+0000 mon.a (mon.0) 709 : cluster [DBG] osdmap e565: 8 total, 8 up, 8 in 2024-09-13T00:44:24.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:23 smithi077 bash[80681]: cluster 2024-09-13T00:44:22.462574+0000 mgr.y (mgr.25304) 235 : cluster [DBG] pgmap v440: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 180 KiB/s rd, 19 KiB/s wr, 247 op/s 2024-09-13T00:44:24.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:23 smithi077 bash[80681]: cluster 2024-09-13T00:44:22.824408+0000 mon.a (mon.0) 710 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:44:24.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:23 smithi077 bash[80681]: cluster 2024-09-13T00:44:22.844353+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e566: 8 total, 8 up, 8 in 2024-09-13T00:44:24.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:23 smithi077 bash[83127]: cluster 2024-09-13T00:44:22.462574+0000 mgr.y (mgr.25304) 235 : cluster [DBG] pgmap v440: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 180 KiB/s rd, 19 KiB/s wr, 247 op/s 2024-09-13T00:44:24.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:23 smithi077 bash[83127]: cluster 2024-09-13T00:44:22.824408+0000 mon.a (mon.0) 710 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:44:24.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:23 smithi077 bash[83127]: cluster 2024-09-13T00:44:22.844353+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e566: 8 total, 8 up, 8 in 2024-09-13T00:44:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:23 smithi088 bash[68960]: cluster 2024-09-13T00:44:22.462574+0000 mgr.y (mgr.25304) 235 : cluster [DBG] pgmap v440: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 180 KiB/s rd, 19 KiB/s wr, 247 op/s 2024-09-13T00:44:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:23 smithi088 bash[68960]: cluster 2024-09-13T00:44:22.824408+0000 mon.a (mon.0) 710 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:44:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:23 smithi088 bash[68960]: cluster 2024-09-13T00:44:22.844353+0000 mon.a (mon.0) 711 : cluster [DBG] osdmap e566: 8 total, 8 up, 8 in 2024-09-13T00:44:24.865 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:25.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:24 smithi077 bash[80681]: cluster 2024-09-13T00:44:23.850141+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e567: 8 total, 8 up, 8 in 2024-09-13T00:44:25.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:24 smithi077 bash[80681]: audit 2024-09-13T00:44:23.853003+0000 mon.a (mon.0) 713 : audit [INF] from='client.? 172.21.15.77:0/251696320' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:25.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:24 smithi077 bash[80681]: audit 2024-09-13T00:44:24.854381+0000 mon.a (mon.0) 714 : audit [INF] from='client.? 172.21.15.77:0/251696320' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:25.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:24 smithi077 bash[80681]: cluster 2024-09-13T00:44:24.855147+0000 mon.a (mon.0) 715 : cluster [DBG] osdmap e568: 8 total, 8 up, 8 in 2024-09-13T00:44:25.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:24 smithi077 bash[83127]: cluster 2024-09-13T00:44:23.850141+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e567: 8 total, 8 up, 8 in 2024-09-13T00:44:25.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:24 smithi077 bash[83127]: audit 2024-09-13T00:44:23.853003+0000 mon.a (mon.0) 713 : audit [INF] from='client.? 172.21.15.77:0/251696320' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:25.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:24 smithi077 bash[83127]: audit 2024-09-13T00:44:24.854381+0000 mon.a (mon.0) 714 : audit [INF] from='client.? 172.21.15.77:0/251696320' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:25.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:24 smithi077 bash[83127]: cluster 2024-09-13T00:44:24.855147+0000 mon.a (mon.0) 715 : cluster [DBG] osdmap e568: 8 total, 8 up, 8 in 2024-09-13T00:44:25.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:24 smithi088 bash[68960]: cluster 2024-09-13T00:44:23.850141+0000 mon.a (mon.0) 712 : cluster [DBG] osdmap e567: 8 total, 8 up, 8 in 2024-09-13T00:44:25.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:24 smithi088 bash[68960]: audit 2024-09-13T00:44:23.853003+0000 mon.a (mon.0) 713 : audit [INF] from='client.? 172.21.15.77:0/251696320' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:25.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:24 smithi088 bash[68960]: audit 2024-09-13T00:44:24.854381+0000 mon.a (mon.0) 714 : audit [INF] from='client.? 172.21.15.77:0/251696320' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:25.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:24 smithi088 bash[68960]: cluster 2024-09-13T00:44:24.855147+0000 mon.a (mon.0) 715 : cluster [DBG] osdmap e568: 8 total, 8 up, 8 in 2024-09-13T00:44:25.978 INFO:tasks.workunit.client.0.smithi077.stdout:num images is: 1 2024-09-13T00:44:25.978 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 1 2024-09-13T00:44:25.978 INFO:tasks.workunit.client.0.smithi077.stdout:image: image20 2024-09-13T00:44:25.978 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image20 2024-09-13T00:44:25.978 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:26.013 INFO:tasks.workunit.client.0.smithi077.stdout:num images is: 2 2024-09-13T00:44:26.013 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 2 2024-09-13T00:44:26.013 INFO:tasks.workunit.client.0.smithi077.stdout:image: image20 2024-09-13T00:44:26.013 INFO:tasks.workunit.client.0.smithi077.stdout:image: image21 2024-09-13T00:44:26.013 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image20 2024-09-13T00:44:26.013 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image21 2024-09-13T00:44:26.165 INFO:tasks.workunit.client.0.smithi077.stdout:num images is: 1 2024-09-13T00:44:26.165 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 1 2024-09-13T00:44:26.165 INFO:tasks.workunit.client.0.smithi077.stdout:image: image21 2024-09-13T00:44:26.165 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image21 2024-09-13T00:44:26.165 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCreateLsDeletePP (3081 ms) 2024-09-13T00:44:26.165 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCopy 2024-09-13T00:44:26.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:25 smithi077 bash[80681]: cluster 2024-09-13T00:44:24.464178+0000 mgr.y (mgr.25304) 236 : cluster [DBG] pgmap v443: 169 pgs: 29 creating+peering, 3 unknown, 137 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 6.5 KiB/s wr, 60 op/s 2024-09-13T00:44:26.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:25 smithi077 bash[80681]: cluster 2024-09-13T00:44:24.857816+0000 mon.a (mon.0) 716 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:44:26.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:25 smithi077 bash[83127]: cluster 2024-09-13T00:44:24.464178+0000 mgr.y (mgr.25304) 236 : cluster [DBG] pgmap v443: 169 pgs: 29 creating+peering, 3 unknown, 137 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 6.5 KiB/s wr, 60 op/s 2024-09-13T00:44:26.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:25 smithi077 bash[83127]: cluster 2024-09-13T00:44:24.857816+0000 mon.a (mon.0) 716 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:44:26.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:25 smithi088 bash[68960]: cluster 2024-09-13T00:44:24.464178+0000 mgr.y (mgr.25304) 236 : cluster [DBG] pgmap v443: 169 pgs: 29 creating+peering, 3 unknown, 137 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 6.5 KiB/s wr, 60 op/s 2024-09-13T00:44:26.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:25 smithi088 bash[68960]: cluster 2024-09-13T00:44:24.857816+0000 mon.a (mon.0) 716 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:44:27.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:26 smithi077 bash[80681]: cluster 2024-09-13T00:44:25.888085+0000 mon.a (mon.0) 717 : cluster [DBG] osdmap e569: 8 total, 8 up, 8 in 2024-09-13T00:44:27.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:26 smithi077 bash[80681]: cluster 2024-09-13T00:44:25.930406+0000 mon.a (mon.0) 718 : cluster [DBG] osdmap e570: 8 total, 8 up, 8 in 2024-09-13T00:44:27.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:26 smithi077 bash[83127]: cluster 2024-09-13T00:44:25.888085+0000 mon.a (mon.0) 717 : cluster [DBG] osdmap e569: 8 total, 8 up, 8 in 2024-09-13T00:44:27.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:26 smithi077 bash[83127]: cluster 2024-09-13T00:44:25.930406+0000 mon.a (mon.0) 718 : cluster [DBG] osdmap e570: 8 total, 8 up, 8 in 2024-09-13T00:44:27.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:26 smithi088 bash[68960]: cluster 2024-09-13T00:44:25.888085+0000 mon.a (mon.0) 717 : cluster [DBG] osdmap e569: 8 total, 8 up, 8 in 2024-09-13T00:44:27.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:26 smithi088 bash[68960]: cluster 2024-09-13T00:44:25.930406+0000 mon.a (mon.0) 718 : cluster [DBG] osdmap e570: 8 total, 8 up, 8 in 2024-09-13T00:44:27.958 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:28.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:27 smithi077 bash[80681]: cluster 2024-09-13T00:44:26.465066+0000 mgr.y (mgr.25304) 237 : cluster [DBG] pgmap v447: 169 pgs: 29 creating+peering, 140 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 5.2 KiB/s wr, 51 op/s 2024-09-13T00:44:28.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:27 smithi077 bash[80681]: cluster 2024-09-13T00:44:26.922474+0000 mon.a (mon.0) 719 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:44:28.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:27 smithi077 bash[80681]: cluster 2024-09-13T00:44:26.941583+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e571: 8 total, 8 up, 8 in 2024-09-13T00:44:28.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:27 smithi077 bash[80681]: audit 2024-09-13T00:44:26.948242+0000 mon.c (mon.2) 24 : audit [INF] from='client.? 172.21.15.77:0/2865127002' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:28.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:27 smithi077 bash[80681]: audit 2024-09-13T00:44:26.950072+0000 mon.a (mon.0) 721 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:28.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:27 smithi077 bash[83127]: cluster 2024-09-13T00:44:26.465066+0000 mgr.y (mgr.25304) 237 : cluster [DBG] pgmap v447: 169 pgs: 29 creating+peering, 140 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 5.2 KiB/s wr, 51 op/s 2024-09-13T00:44:28.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:27 smithi077 bash[83127]: cluster 2024-09-13T00:44:26.922474+0000 mon.a (mon.0) 719 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:44:28.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:27 smithi077 bash[83127]: cluster 2024-09-13T00:44:26.941583+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e571: 8 total, 8 up, 8 in 2024-09-13T00:44:28.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:27 smithi077 bash[83127]: audit 2024-09-13T00:44:26.948242+0000 mon.c (mon.2) 24 : audit [INF] from='client.? 172.21.15.77:0/2865127002' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:28.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:27 smithi077 bash[83127]: audit 2024-09-13T00:44:26.950072+0000 mon.a (mon.0) 721 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:27 smithi088 bash[68960]: cluster 2024-09-13T00:44:26.465066+0000 mgr.y (mgr.25304) 237 : cluster [DBG] pgmap v447: 169 pgs: 29 creating+peering, 140 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 5.2 KiB/s wr, 51 op/s 2024-09-13T00:44:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:27 smithi088 bash[68960]: cluster 2024-09-13T00:44:26.922474+0000 mon.a (mon.0) 719 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:44:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:27 smithi088 bash[68960]: cluster 2024-09-13T00:44:26.941583+0000 mon.a (mon.0) 720 : cluster [DBG] osdmap e571: 8 total, 8 up, 8 in 2024-09-13T00:44:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:27 smithi088 bash[68960]: audit 2024-09-13T00:44:26.948242+0000 mon.c (mon.2) 24 : audit [INF] from='client.? 172.21.15.77:0/2865127002' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:27 smithi088 bash[68960]: audit 2024-09-13T00:44:26.950072+0000 mon.a (mon.0) 721 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:28.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:44:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:44:28] "GET /metrics HTTP/1.1" 200 43704 "" "Prometheus/2.43.0" 2024-09-13T00:44:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:28 smithi077 bash[80681]: audit 2024-09-13T00:44:27.942268+0000 mon.a (mon.0) 722 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:28 smithi077 bash[80681]: cluster 2024-09-13T00:44:27.942730+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e572: 8 total, 8 up, 8 in 2024-09-13T00:44:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:28 smithi077 bash[80681]: audit 2024-09-13T00:44:28.416274+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:29.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:28 smithi077 bash[83127]: audit 2024-09-13T00:44:27.942268+0000 mon.a (mon.0) 722 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:29.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:28 smithi077 bash[83127]: cluster 2024-09-13T00:44:27.942730+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e572: 8 total, 8 up, 8 in 2024-09-13T00:44:29.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:28 smithi077 bash[83127]: audit 2024-09-13T00:44:28.416274+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:28 smithi088 bash[68960]: audit 2024-09-13T00:44:27.942268+0000 mon.a (mon.0) 722 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:28 smithi088 bash[68960]: cluster 2024-09-13T00:44:27.942730+0000 mon.a (mon.0) 723 : cluster [DBG] osdmap e572: 8 total, 8 up, 8 in 2024-09-13T00:44:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:28 smithi088 bash[68960]: audit 2024-09-13T00:44:28.416274+0000 mon.a (mon.0) 724 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:30.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:29 smithi077 bash[80681]: cluster 2024-09-13T00:44:28.465906+0000 mgr.y (mgr.25304) 238 : cluster [DBG] pgmap v450: 201 pgs: 32 unknown, 29 creating+peering, 140 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s rd, 849 B/s wr, 2 op/s 2024-09-13T00:44:30.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:29 smithi077 bash[80681]: cluster 2024-09-13T00:44:28.954061+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e573: 8 total, 8 up, 8 in 2024-09-13T00:44:30.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:29 smithi077 bash[83127]: cluster 2024-09-13T00:44:28.465906+0000 mgr.y (mgr.25304) 238 : cluster [DBG] pgmap v450: 201 pgs: 32 unknown, 29 creating+peering, 140 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s rd, 849 B/s wr, 2 op/s 2024-09-13T00:44:30.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:29 smithi077 bash[83127]: cluster 2024-09-13T00:44:28.954061+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e573: 8 total, 8 up, 8 in 2024-09-13T00:44:30.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:29 smithi088 bash[68960]: cluster 2024-09-13T00:44:28.465906+0000 mgr.y (mgr.25304) 238 : cluster [DBG] pgmap v450: 201 pgs: 32 unknown, 29 creating+peering, 140 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.4 KiB/s rd, 849 B/s wr, 2 op/s 2024-09-13T00:44:30.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:29 smithi088 bash[68960]: cluster 2024-09-13T00:44:28.954061+0000 mon.a (mon.0) 725 : cluster [DBG] osdmap e573: 8 total, 8 up, 8 in 2024-09-13T00:44:31.014 INFO:tasks.workunit.client.0.smithi077.stdout:image: image22 2024-09-13T00:44:31.014 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image22 2024-09-13T00:44:31.014 INFO:tasks.workunit.client.0.smithi077.stdout:found image22 2024-09-13T00:44:31.014 INFO:tasks.workunit.client.0.smithi077.stdout:erased image22 2024-09-13T00:44:31.015 INFO:tasks.workunit.client.0.smithi077.stdout:image: image22 2024-09-13T00:44:31.015 INFO:tasks.workunit.client.0.smithi077.stdout:image: image23 2024-09-13T00:44:31.015 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image22 2024-09-13T00:44:31.015 INFO:tasks.workunit.client.0.smithi077.stdout:found image22 2024-09-13T00:44:31.015 INFO:tasks.workunit.client.0.smithi077.stdout:erased image22 2024-09-13T00:44:31.015 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image23 2024-09-13T00:44:31.015 INFO:tasks.workunit.client.0.smithi077.stdout:found image23 2024-09-13T00:44:31.015 INFO:tasks.workunit.client.0.smithi077.stdout:erased image23 2024-09-13T00:44:31.015 INFO:tasks.workunit.client.0.smithi077.stdout:100.00% done 2024-09-13T00:44:31.015 INFO:tasks.workunit.client.0.smithi077.stdout:image: image22 2024-09-13T00:44:31.015 INFO:tasks.workunit.client.0.smithi077.stdout:image: image23 2024-09-13T00:44:31.015 INFO:tasks.workunit.client.0.smithi077.stdout:image: image24 2024-09-13T00:44:31.015 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image22 2024-09-13T00:44:31.015 INFO:tasks.workunit.client.0.smithi077.stdout:found image22 2024-09-13T00:44:31.016 INFO:tasks.workunit.client.0.smithi077.stdout:erased image22 2024-09-13T00:44:31.016 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image23 2024-09-13T00:44:31.016 INFO:tasks.workunit.client.0.smithi077.stdout:found image23 2024-09-13T00:44:31.016 INFO:tasks.workunit.client.0.smithi077.stdout:erased image23 2024-09-13T00:44:31.016 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image24 2024-09-13T00:44:31.016 INFO:tasks.workunit.client.0.smithi077.stdout:found image24 2024-09-13T00:44:31.016 INFO:tasks.workunit.client.0.smithi077.stdout:erased image24 2024-09-13T00:44:31.016 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCopy (4850 ms) 2024-09-13T00:44:31.016 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCopyPP 2024-09-13T00:44:31.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:30 smithi077 bash[80681]: cluster 2024-09-13T00:44:29.963875+0000 mon.a (mon.0) 726 : cluster [DBG] osdmap e574: 8 total, 8 up, 8 in 2024-09-13T00:44:31.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:30 smithi077 bash[83127]: cluster 2024-09-13T00:44:29.963875+0000 mon.a (mon.0) 726 : cluster [DBG] osdmap e574: 8 total, 8 up, 8 in 2024-09-13T00:44:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:30 smithi088 bash[68960]: cluster 2024-09-13T00:44:29.963875+0000 mon.a (mon.0) 726 : cluster [DBG] osdmap e574: 8 total, 8 up, 8 in 2024-09-13T00:44:32.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:31 smithi077 bash[80681]: cluster 2024-09-13T00:44:30.466734+0000 mgr.y (mgr.25304) 239 : cluster [DBG] pgmap v453: 201 pgs: 3 unknown, 198 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.7 KiB/s wr, 38 op/s 2024-09-13T00:44:32.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:31 smithi077 bash[83127]: cluster 2024-09-13T00:44:30.466734+0000 mgr.y (mgr.25304) 239 : cluster [DBG] pgmap v453: 201 pgs: 3 unknown, 198 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.7 KiB/s wr, 38 op/s 2024-09-13T00:44:32.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:31 smithi088 bash[68960]: cluster 2024-09-13T00:44:30.466734+0000 mgr.y (mgr.25304) 239 : cluster [DBG] pgmap v453: 201 pgs: 3 unknown, 198 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.7 KiB/s wr, 38 op/s 2024-09-13T00:44:32.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:44:32 smithi088 bash[61791]: ts=2024-09-13T00:44:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:44:33.021 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:33.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:33 smithi077 bash[80681]: cluster 2024-09-13T00:44:31.978944+0000 mon.a (mon.0) 727 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2024-09-13T00:44:33.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:33 smithi077 bash[80681]: audit 2024-09-13T00:44:31.988732+0000 mon.c (mon.2) 25 : audit [INF] from='client.? 172.21.15.77:0/1718358639' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:33.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:33 smithi077 bash[80681]: audit 2024-09-13T00:44:31.998673+0000 mon.a (mon.0) 728 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:33.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:33 smithi077 bash[80681]: audit 2024-09-13T00:44:32.980592+0000 mon.a (mon.0) 729 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:33.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:33 smithi077 bash[80681]: cluster 2024-09-13T00:44:32.980852+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2024-09-13T00:44:33.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:33 smithi077 bash[83127]: cluster 2024-09-13T00:44:31.978944+0000 mon.a (mon.0) 727 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2024-09-13T00:44:33.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:33 smithi077 bash[83127]: audit 2024-09-13T00:44:31.988732+0000 mon.c (mon.2) 25 : audit [INF] from='client.? 172.21.15.77:0/1718358639' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:33.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:33 smithi077 bash[83127]: audit 2024-09-13T00:44:31.998673+0000 mon.a (mon.0) 728 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:33.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:33 smithi077 bash[83127]: audit 2024-09-13T00:44:32.980592+0000 mon.a (mon.0) 729 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:33.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:33 smithi077 bash[83127]: cluster 2024-09-13T00:44:32.980852+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2024-09-13T00:44:33.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:33 smithi088 bash[68960]: cluster 2024-09-13T00:44:31.978944+0000 mon.a (mon.0) 727 : cluster [DBG] osdmap e575: 8 total, 8 up, 8 in 2024-09-13T00:44:33.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:33 smithi088 bash[68960]: audit 2024-09-13T00:44:31.988732+0000 mon.c (mon.2) 25 : audit [INF] from='client.? 172.21.15.77:0/1718358639' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:33.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:33 smithi088 bash[68960]: audit 2024-09-13T00:44:31.998673+0000 mon.a (mon.0) 728 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:33.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:33 smithi088 bash[68960]: audit 2024-09-13T00:44:32.980592+0000 mon.a (mon.0) 729 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:33.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:33 smithi088 bash[68960]: cluster 2024-09-13T00:44:32.980852+0000 mon.a (mon.0) 730 : cluster [DBG] osdmap e576: 8 total, 8 up, 8 in 2024-09-13T00:44:34.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:34 smithi077 bash[80681]: cluster 2024-09-13T00:44:32.467503+0000 mgr.y (mgr.25304) 240 : cluster [DBG] pgmap v455: 233 pgs: 32 unknown, 201 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.4 KiB/s wr, 34 op/s 2024-09-13T00:44:34.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:34 smithi077 bash[80681]: cluster 2024-09-13T00:44:32.988591+0000 mon.a (mon.0) 731 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:44:34.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:34 smithi077 bash[80681]: cluster 2024-09-13T00:44:33.981120+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2024-09-13T00:44:34.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:34 smithi077 bash[83127]: cluster 2024-09-13T00:44:32.467503+0000 mgr.y (mgr.25304) 240 : cluster [DBG] pgmap v455: 233 pgs: 32 unknown, 201 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.4 KiB/s wr, 34 op/s 2024-09-13T00:44:34.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:34 smithi077 bash[83127]: cluster 2024-09-13T00:44:32.988591+0000 mon.a (mon.0) 731 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:44:34.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:34 smithi077 bash[83127]: cluster 2024-09-13T00:44:33.981120+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2024-09-13T00:44:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:34 smithi088 bash[68960]: cluster 2024-09-13T00:44:32.467503+0000 mgr.y (mgr.25304) 240 : cluster [DBG] pgmap v455: 233 pgs: 32 unknown, 201 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 2.4 KiB/s wr, 34 op/s 2024-09-13T00:44:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:34 smithi088 bash[68960]: cluster 2024-09-13T00:44:32.988591+0000 mon.a (mon.0) 731 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:44:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:34 smithi088 bash[68960]: cluster 2024-09-13T00:44:33.981120+0000 mon.a (mon.0) 732 : cluster [DBG] osdmap e577: 8 total, 8 up, 8 in 2024-09-13T00:44:35.696 INFO:tasks.workunit.client.0.smithi077.stdout:num images is: 1 2024-09-13T00:44:35.696 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 1 2024-09-13T00:44:35.696 INFO:tasks.workunit.client.0.smithi077.stdout:image: image25 2024-09-13T00:44:35.696 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image25 2024-09-13T00:44:35.751 INFO:tasks.workunit.client.0.smithi077.stdout:num images is: 2 2024-09-13T00:44:35.751 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 2 2024-09-13T00:44:35.751 INFO:tasks.workunit.client.0.smithi077.stdout:image: image25 2024-09-13T00:44:35.751 INFO:tasks.workunit.client.0.smithi077.stdout:image: image26 2024-09-13T00:44:35.751 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image25 2024-09-13T00:44:35.751 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image26 2024-09-13T00:44:35.874 INFO:tasks.workunit.client.0.smithi077.stdout:100.00% done 2024-09-13T00:44:35.874 INFO:tasks.workunit.client.0.smithi077.stdout:num images is: 3 2024-09-13T00:44:35.874 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 3 2024-09-13T00:44:35.874 INFO:tasks.workunit.client.0.smithi077.stdout:image: image25 2024-09-13T00:44:35.874 INFO:tasks.workunit.client.0.smithi077.stdout:image: image26 2024-09-13T00:44:35.874 INFO:tasks.workunit.client.0.smithi077.stdout:image: image27 2024-09-13T00:44:35.875 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image25 2024-09-13T00:44:35.875 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image26 2024-09-13T00:44:35.875 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image27 2024-09-13T00:44:36.006 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCopyPP (4991 ms) 2024-09-13T00:44:36.006 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestDeepCopy 2024-09-13T00:44:36.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:35 smithi077 bash[80681]: cluster 2024-09-13T00:44:34.469520+0000 mgr.y (mgr.25304) 241 : cluster [DBG] pgmap v458: 233 pgs: 1 unknown, 232 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 303 KiB/s rd, 20 KiB/s wr, 354 op/s 2024-09-13T00:44:36.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:35 smithi077 bash[80681]: cluster 2024-09-13T00:44:34.984551+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2024-09-13T00:44:36.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:35 smithi077 bash[80681]: cluster 2024-09-13T00:44:35.022819+0000 mon.a (mon.0) 734 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:44:36.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:35 smithi077 bash[80681]: cluster 2024-09-13T00:44:35.935291+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2024-09-13T00:44:36.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:35 smithi077 bash[83127]: cluster 2024-09-13T00:44:34.469520+0000 mgr.y (mgr.25304) 241 : cluster [DBG] pgmap v458: 233 pgs: 1 unknown, 232 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 303 KiB/s rd, 20 KiB/s wr, 354 op/s 2024-09-13T00:44:36.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:35 smithi077 bash[83127]: cluster 2024-09-13T00:44:34.984551+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2024-09-13T00:44:36.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:35 smithi077 bash[83127]: cluster 2024-09-13T00:44:35.022819+0000 mon.a (mon.0) 734 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:44:36.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:35 smithi077 bash[83127]: cluster 2024-09-13T00:44:35.935291+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2024-09-13T00:44:36.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:35 smithi088 bash[68960]: cluster 2024-09-13T00:44:34.469520+0000 mgr.y (mgr.25304) 241 : cluster [DBG] pgmap v458: 233 pgs: 1 unknown, 232 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 303 KiB/s rd, 20 KiB/s wr, 354 op/s 2024-09-13T00:44:36.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:35 smithi088 bash[68960]: cluster 2024-09-13T00:44:34.984551+0000 mon.a (mon.0) 733 : cluster [DBG] osdmap e578: 8 total, 8 up, 8 in 2024-09-13T00:44:36.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:35 smithi088 bash[68960]: cluster 2024-09-13T00:44:35.022819+0000 mon.a (mon.0) 734 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:44:36.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:35 smithi088 bash[68960]: cluster 2024-09-13T00:44:35.935291+0000 mon.a (mon.0) 735 : cluster [DBG] osdmap e579: 8 total, 8 up, 8 in 2024-09-13T00:44:38.019 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:38.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:37 smithi077 bash[80681]: cluster 2024-09-13T00:44:36.470554+0000 mgr.y (mgr.25304) 242 : cluster [DBG] pgmap v461: 233 pgs: 233 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 398 op/s 2024-09-13T00:44:38.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:37 smithi077 bash[80681]: cluster 2024-09-13T00:44:36.997284+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2024-09-13T00:44:38.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:37 smithi077 bash[80681]: audit 2024-09-13T00:44:37.005771+0000 mon.c (mon.2) 26 : audit [INF] from='client.? 172.21.15.77:0/1867831023' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:38.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:37 smithi077 bash[80681]: audit 2024-09-13T00:44:37.007228+0000 mon.a (mon.0) 737 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:38.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:37 smithi077 bash[83127]: cluster 2024-09-13T00:44:36.470554+0000 mgr.y (mgr.25304) 242 : cluster [DBG] pgmap v461: 233 pgs: 233 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 398 op/s 2024-09-13T00:44:38.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:37 smithi077 bash[83127]: cluster 2024-09-13T00:44:36.997284+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2024-09-13T00:44:38.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:37 smithi077 bash[83127]: audit 2024-09-13T00:44:37.005771+0000 mon.c (mon.2) 26 : audit [INF] from='client.? 172.21.15.77:0/1867831023' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:38.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:37 smithi077 bash[83127]: audit 2024-09-13T00:44:37.007228+0000 mon.a (mon.0) 737 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:38.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:37 smithi088 bash[68960]: cluster 2024-09-13T00:44:36.470554+0000 mgr.y (mgr.25304) 242 : cluster [DBG] pgmap v461: 233 pgs: 233 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 398 op/s 2024-09-13T00:44:38.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:37 smithi088 bash[68960]: cluster 2024-09-13T00:44:36.997284+0000 mon.a (mon.0) 736 : cluster [DBG] osdmap e580: 8 total, 8 up, 8 in 2024-09-13T00:44:38.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:37 smithi088 bash[68960]: audit 2024-09-13T00:44:37.005771+0000 mon.c (mon.2) 26 : audit [INF] from='client.? 172.21.15.77:0/1867831023' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:38.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:37 smithi088 bash[68960]: audit 2024-09-13T00:44:37.007228+0000 mon.a (mon.0) 737 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:38.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:44:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:44:38] "GET /metrics HTTP/1.1" 200 43704 "" "Prometheus/2.43.0" 2024-09-13T00:44:39.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:39 smithi077 bash[80681]: audit 2024-09-13T00:44:38.009190+0000 mon.a (mon.0) 738 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:39.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:39 smithi077 bash[80681]: cluster 2024-09-13T00:44:38.009641+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2024-09-13T00:44:39.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:39 smithi077 bash[83127]: audit 2024-09-13T00:44:38.009190+0000 mon.a (mon.0) 738 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:39.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:39 smithi077 bash[83127]: cluster 2024-09-13T00:44:38.009641+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2024-09-13T00:44:39.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:39 smithi088 bash[68960]: audit 2024-09-13T00:44:38.009190+0000 mon.a (mon.0) 738 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:39.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:39 smithi088 bash[68960]: cluster 2024-09-13T00:44:38.009641+0000 mon.a (mon.0) 739 : cluster [DBG] osdmap e581: 8 total, 8 up, 8 in 2024-09-13T00:44:40.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:40 smithi077 bash[80681]: cluster 2024-09-13T00:44:38.471331+0000 mgr.y (mgr.25304) 243 : cluster [DBG] pgmap v464: 265 pgs: 32 unknown, 233 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:40.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:40 smithi077 bash[80681]: cluster 2024-09-13T00:44:39.021235+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2024-09-13T00:44:40.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:40 smithi077 bash[83127]: cluster 2024-09-13T00:44:38.471331+0000 mgr.y (mgr.25304) 243 : cluster [DBG] pgmap v464: 265 pgs: 32 unknown, 233 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:40.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:40 smithi077 bash[83127]: cluster 2024-09-13T00:44:39.021235+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2024-09-13T00:44:40.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:40 smithi088 bash[68960]: cluster 2024-09-13T00:44:38.471331+0000 mgr.y (mgr.25304) 243 : cluster [DBG] pgmap v464: 265 pgs: 32 unknown, 233 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:44:40.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:40 smithi088 bash[68960]: cluster 2024-09-13T00:44:39.021235+0000 mon.a (mon.0) 740 : cluster [DBG] osdmap e582: 8 total, 8 up, 8 in 2024-09-13T00:44:41.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:41 smithi088 bash[68960]: cluster 2024-09-13T00:44:40.031560+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2024-09-13T00:44:41.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:41 smithi077 bash[80681]: cluster 2024-09-13T00:44:40.031560+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2024-09-13T00:44:41.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:41 smithi077 bash[83127]: cluster 2024-09-13T00:44:40.031560+0000 mon.a (mon.0) 741 : cluster [DBG] osdmap e583: 8 total, 8 up, 8 in 2024-09-13T00:44:42.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:42 smithi088 bash[68960]: cluster 2024-09-13T00:44:40.472877+0000 mgr.y (mgr.25304) 244 : cluster [DBG] pgmap v467: 265 pgs: 7 unknown, 258 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 397 op/s 2024-09-13T00:44:42.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:42 smithi088 bash[68960]: cluster 2024-09-13T00:44:41.039927+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2024-09-13T00:44:42.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:42 smithi077 bash[80681]: cluster 2024-09-13T00:44:40.472877+0000 mgr.y (mgr.25304) 244 : cluster [DBG] pgmap v467: 265 pgs: 7 unknown, 258 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 397 op/s 2024-09-13T00:44:42.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:42 smithi077 bash[80681]: cluster 2024-09-13T00:44:41.039927+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2024-09-13T00:44:42.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:42 smithi077 bash[83127]: cluster 2024-09-13T00:44:40.472877+0000 mgr.y (mgr.25304) 244 : cluster [DBG] pgmap v467: 265 pgs: 7 unknown, 258 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 341 KiB/s rd, 22 KiB/s wr, 397 op/s 2024-09-13T00:44:42.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:42 smithi077 bash[83127]: cluster 2024-09-13T00:44:41.039927+0000 mon.a (mon.0) 742 : cluster [DBG] osdmap e584: 8 total, 8 up, 8 in 2024-09-13T00:44:42.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:44:42 smithi088 bash[61791]: ts=2024-09-13T00:44:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:44:43.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:43 smithi088 bash[68960]: cluster 2024-09-13T00:44:42.049681+0000 mon.a (mon.0) 743 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2024-09-13T00:44:43.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:43 smithi077 bash[80681]: cluster 2024-09-13T00:44:42.049681+0000 mon.a (mon.0) 743 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2024-09-13T00:44:43.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:43 smithi077 bash[83127]: cluster 2024-09-13T00:44:42.049681+0000 mon.a (mon.0) 743 : cluster [DBG] osdmap e585: 8 total, 8 up, 8 in 2024-09-13T00:44:44.253 INFO:tasks.workunit.client.0.smithi077.stdout:image: image28 2024-09-13T00:44:44.253 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image28 2024-09-13T00:44:44.253 INFO:tasks.workunit.client.0.smithi077.stdout:found image28 2024-09-13T00:44:44.253 INFO:tasks.workunit.client.0.smithi077.stdout:erased image28 2024-09-13T00:44:44.253 INFO:tasks.workunit.client.0.smithi077.stdout:image: image28 2024-09-13T00:44:44.253 INFO:tasks.workunit.client.0.smithi077.stdout:image: image29 2024-09-13T00:44:44.253 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image28 2024-09-13T00:44:44.253 INFO:tasks.workunit.client.0.smithi077.stdout:found image28 2024-09-13T00:44:44.254 INFO:tasks.workunit.client.0.smithi077.stdout:erased image28 2024-09-13T00:44:44.254 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image29 2024-09-13T00:44:44.254 INFO:tasks.workunit.client.0.smithi077.stdout:found image29 2024-09-13T00:44:44.254 INFO:tasks.workunit.client.0.smithi077.stdout:erased image29 2024-09-13T00:44:44.254 INFO:tasks.workunit.client.0.smithi077.stdout:100.00% done 2024-09-13T00:44:44.254 INFO:tasks.workunit.client.0.smithi077.stdout:image: image28 2024-09-13T00:44:44.254 INFO:tasks.workunit.client.0.smithi077.stdout:image: image29 2024-09-13T00:44:44.254 INFO:tasks.workunit.client.0.smithi077.stdout:image: image30 2024-09-13T00:44:44.254 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image28 2024-09-13T00:44:44.254 INFO:tasks.workunit.client.0.smithi077.stdout:found image28 2024-09-13T00:44:44.254 INFO:tasks.workunit.client.0.smithi077.stdout:erased image28 2024-09-13T00:44:44.254 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image29 2024-09-13T00:44:44.254 INFO:tasks.workunit.client.0.smithi077.stdout:found image29 2024-09-13T00:44:44.255 INFO:tasks.workunit.client.0.smithi077.stdout:erased image29 2024-09-13T00:44:44.255 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image30 2024-09-13T00:44:44.255 INFO:tasks.workunit.client.0.smithi077.stdout:found image30 2024-09-13T00:44:44.255 INFO:tasks.workunit.client.0.smithi077.stdout:erased image30 2024-09-13T00:44:44.255 INFO:tasks.workunit.client.0.smithi077.stdout:image: image28 2024-09-13T00:44:44.255 INFO:tasks.workunit.client.0.smithi077.stdout:image: image29 2024-09-13T00:44:44.255 INFO:tasks.workunit.client.0.smithi077.stdout:image: image30 2024-09-13T00:44:44.255 INFO:tasks.workunit.client.0.smithi077.stdout:image: image31 2024-09-13T00:44:44.255 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image28 2024-09-13T00:44:44.255 INFO:tasks.workunit.client.0.smithi077.stdout:found image28 2024-09-13T00:44:44.255 INFO:tasks.workunit.client.0.smithi077.stdout:erased image28 2024-09-13T00:44:44.255 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image29 2024-09-13T00:44:44.255 INFO:tasks.workunit.client.0.smithi077.stdout:found image29 2024-09-13T00:44:44.255 INFO:tasks.workunit.client.0.smithi077.stdout:erased image29 2024-09-13T00:44:44.256 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image30 2024-09-13T00:44:44.256 INFO:tasks.workunit.client.0.smithi077.stdout:found image30 2024-09-13T00:44:44.256 INFO:tasks.workunit.client.0.smithi077.stdout:erased image30 2024-09-13T00:44:44.256 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image31 2024-09-13T00:44:44.256 INFO:tasks.workunit.client.0.smithi077.stdout:found image31 2024-09-13T00:44:44.256 INFO:tasks.workunit.client.0.smithi077.stdout:erased image31 2024-09-13T00:44:44.256 INFO:tasks.workunit.client.0.smithi077.stdout:image: image28 2024-09-13T00:44:44.256 INFO:tasks.workunit.client.0.smithi077.stdout:image: image29 2024-09-13T00:44:44.256 INFO:tasks.workunit.client.0.smithi077.stdout:image: image30 2024-09-13T00:44:44.256 INFO:tasks.workunit.client.0.smithi077.stdout:image: image31 2024-09-13T00:44:44.256 INFO:tasks.workunit.client.0.smithi077.stdout:image: image32 2024-09-13T00:44:44.256 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image28 2024-09-13T00:44:44.256 INFO:tasks.workunit.client.0.smithi077.stdout:found image28 2024-09-13T00:44:44.256 INFO:tasks.workunit.client.0.smithi077.stdout:erased image28 2024-09-13T00:44:44.257 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image29 2024-09-13T00:44:44.257 INFO:tasks.workunit.client.0.smithi077.stdout:found image29 2024-09-13T00:44:44.257 INFO:tasks.workunit.client.0.smithi077.stdout:erased image29 2024-09-13T00:44:44.257 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image30 2024-09-13T00:44:44.257 INFO:tasks.workunit.client.0.smithi077.stdout:found image30 2024-09-13T00:44:44.257 INFO:tasks.workunit.client.0.smithi077.stdout:erased image30 2024-09-13T00:44:44.257 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image31 2024-09-13T00:44:44.257 INFO:tasks.workunit.client.0.smithi077.stdout:found image31 2024-09-13T00:44:44.257 INFO:tasks.workunit.client.0.smithi077.stdout:erased image31 2024-09-13T00:44:44.257 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image32 2024-09-13T00:44:44.257 INFO:tasks.workunit.client.0.smithi077.stdout:found image32 2024-09-13T00:44:44.257 INFO:tasks.workunit.client.0.smithi077.stdout:erased image32 2024-09-13T00:44:44.257 INFO:tasks.workunit.client.0.smithi077.stdout:100.00% done 2024-09-13T00:44:44.257 INFO:tasks.workunit.client.0.smithi077.stdout:image: image28 2024-09-13T00:44:44.258 INFO:tasks.workunit.client.0.smithi077.stdout:image: image29 2024-09-13T00:44:44.258 INFO:tasks.workunit.client.0.smithi077.stdout:image: image30 2024-09-13T00:44:44.258 INFO:tasks.workunit.client.0.smithi077.stdout:image: image31 2024-09-13T00:44:44.258 INFO:tasks.workunit.client.0.smithi077.stdout:image: image32 2024-09-13T00:44:44.258 INFO:tasks.workunit.client.0.smithi077.stdout:image: image33 2024-09-13T00:44:44.258 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image28 2024-09-13T00:44:44.258 INFO:tasks.workunit.client.0.smithi077.stdout:found image28 2024-09-13T00:44:44.258 INFO:tasks.workunit.client.0.smithi077.stdout:erased image28 2024-09-13T00:44:44.258 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image29 2024-09-13T00:44:44.258 INFO:tasks.workunit.client.0.smithi077.stdout:found image29 2024-09-13T00:44:44.258 INFO:tasks.workunit.client.0.smithi077.stdout:erased image29 2024-09-13T00:44:44.258 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image30 2024-09-13T00:44:44.258 INFO:tasks.workunit.client.0.smithi077.stdout:found image30 2024-09-13T00:44:44.258 INFO:tasks.workunit.client.0.smithi077.stdout:erased image30 2024-09-13T00:44:44.259 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image31 2024-09-13T00:44:44.259 INFO:tasks.workunit.client.0.smithi077.stdout:found image31 2024-09-13T00:44:44.259 INFO:tasks.workunit.client.0.smithi077.stdout:erased image31 2024-09-13T00:44:44.259 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image32 2024-09-13T00:44:44.259 INFO:tasks.workunit.client.0.smithi077.stdout:found image32 2024-09-13T00:44:44.259 INFO:tasks.workunit.client.0.smithi077.stdout:erased image32 2024-09-13T00:44:44.259 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image33 2024-09-13T00:44:44.259 INFO:tasks.workunit.client.0.smithi077.stdout:found image33 2024-09-13T00:44:44.259 INFO:tasks.workunit.client.0.smithi077.stdout:erased image33 2024-09-13T00:44:44.259 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestDeepCopy (8236 ms) 2024-09-13T00:44:44.259 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestDeepCopyPP 2024-09-13T00:44:44.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:44 smithi077 bash[80681]: cluster 2024-09-13T00:44:42.473696+0000 mgr.y (mgr.25304) 245 : cluster [DBG] pgmap v470: 265 pgs: 265 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 364 KiB/s rd, 22 KiB/s wr, 423 op/s 2024-09-13T00:44:44.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:44 smithi077 bash[80681]: cluster 2024-09-13T00:44:43.059399+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2024-09-13T00:44:44.533 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:44 smithi077 bash[80681]: audit 2024-09-13T00:44:43.416748+0000 mon.a (mon.0) 745 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:44.533 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:44 smithi077 bash[83127]: cluster 2024-09-13T00:44:42.473696+0000 mgr.y (mgr.25304) 245 : cluster [DBG] pgmap v470: 265 pgs: 265 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 364 KiB/s rd, 22 KiB/s wr, 423 op/s 2024-09-13T00:44:44.533 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:44 smithi077 bash[83127]: cluster 2024-09-13T00:44:43.059399+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2024-09-13T00:44:44.533 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:44 smithi077 bash[83127]: audit 2024-09-13T00:44:43.416748+0000 mon.a (mon.0) 745 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:44.559 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:44 smithi088 bash[68960]: cluster 2024-09-13T00:44:42.473696+0000 mgr.y (mgr.25304) 245 : cluster [DBG] pgmap v470: 265 pgs: 265 active+clean; 583 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 364 KiB/s rd, 22 KiB/s wr, 423 op/s 2024-09-13T00:44:44.559 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:44 smithi088 bash[68960]: cluster 2024-09-13T00:44:43.059399+0000 mon.a (mon.0) 744 : cluster [DBG] osdmap e586: 8 total, 8 up, 8 in 2024-09-13T00:44:44.559 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:44 smithi088 bash[68960]: audit 2024-09-13T00:44:43.416748+0000 mon.a (mon.0) 745 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:45.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:45 smithi088 bash[68960]: cluster 2024-09-13T00:44:44.069209+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2024-09-13T00:44:45.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:45 smithi077 bash[80681]: cluster 2024-09-13T00:44:44.069209+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2024-09-13T00:44:45.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:45 smithi077 bash[83127]: cluster 2024-09-13T00:44:44.069209+0000 mon.a (mon.0) 746 : cluster [DBG] osdmap e587: 8 total, 8 up, 8 in 2024-09-13T00:44:45.942 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:46.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:46 smithi088 bash[68960]: cluster 2024-09-13T00:44:44.475841+0000 mgr.y (mgr.25304) 246 : cluster [DBG] pgmap v473: 265 pgs: 265 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 541 KiB/s rd, 32 KiB/s wr, 640 op/s 2024-09-13T00:44:46.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:46 smithi088 bash[68960]: cluster 2024-09-13T00:44:45.080523+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2024-09-13T00:44:46.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:46 smithi088 bash[68960]: audit 2024-09-13T00:44:45.085824+0000 mon.c (mon.2) 27 : audit [INF] from='client.? 172.21.15.77:0/3405680500' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:46.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:46 smithi088 bash[68960]: audit 2024-09-13T00:44:45.086765+0000 mon.a (mon.0) 748 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:46.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:46 smithi088 bash[68960]: audit 2024-09-13T00:44:45.939073+0000 mon.a (mon.0) 749 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:46.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:46 smithi088 bash[68960]: cluster 2024-09-13T00:44:45.939490+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2024-09-13T00:44:46.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:46 smithi077 bash[83127]: cluster 2024-09-13T00:44:44.475841+0000 mgr.y (mgr.25304) 246 : cluster [DBG] pgmap v473: 265 pgs: 265 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 541 KiB/s rd, 32 KiB/s wr, 640 op/s 2024-09-13T00:44:46.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:46 smithi077 bash[83127]: cluster 2024-09-13T00:44:45.080523+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2024-09-13T00:44:46.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:46 smithi077 bash[83127]: audit 2024-09-13T00:44:45.085824+0000 mon.c (mon.2) 27 : audit [INF] from='client.? 172.21.15.77:0/3405680500' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:46.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:46 smithi077 bash[83127]: audit 2024-09-13T00:44:45.086765+0000 mon.a (mon.0) 748 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:46.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:46 smithi077 bash[83127]: audit 2024-09-13T00:44:45.939073+0000 mon.a (mon.0) 749 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:46.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:46 smithi077 bash[83127]: cluster 2024-09-13T00:44:45.939490+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2024-09-13T00:44:46.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:46 smithi077 bash[80681]: cluster 2024-09-13T00:44:44.475841+0000 mgr.y (mgr.25304) 246 : cluster [DBG] pgmap v473: 265 pgs: 265 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 541 KiB/s rd, 32 KiB/s wr, 640 op/s 2024-09-13T00:44:46.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:46 smithi077 bash[80681]: cluster 2024-09-13T00:44:45.080523+0000 mon.a (mon.0) 747 : cluster [DBG] osdmap e588: 8 total, 8 up, 8 in 2024-09-13T00:44:46.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:46 smithi077 bash[80681]: audit 2024-09-13T00:44:45.085824+0000 mon.c (mon.2) 27 : audit [INF] from='client.? 172.21.15.77:0/3405680500' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:46.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:46 smithi077 bash[80681]: audit 2024-09-13T00:44:45.086765+0000 mon.a (mon.0) 748 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:44:46.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:46 smithi077 bash[80681]: audit 2024-09-13T00:44:45.939073+0000 mon.a (mon.0) 749 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:44:46.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:46 smithi077 bash[80681]: cluster 2024-09-13T00:44:45.939490+0000 mon.a (mon.0) 750 : cluster [DBG] osdmap e589: 8 total, 8 up, 8 in 2024-09-13T00:44:48.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:47 smithi077 bash[80681]: cluster 2024-09-13T00:44:46.476909+0000 mgr.y (mgr.25304) 247 : cluster [DBG] pgmap v476: 297 pgs: 29 unknown, 268 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 525 KiB/s rd, 31 KiB/s wr, 625 op/s 2024-09-13T00:44:48.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:47 smithi077 bash[80681]: cluster 2024-09-13T00:44:46.940735+0000 mon.a (mon.0) 751 : cluster [DBG] osdmap e590: 8 total, 8 up, 8 in 2024-09-13T00:44:48.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:47 smithi077 bash[80681]: audit 2024-09-13T00:44:47.727936+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:48.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:47 smithi077 bash[83127]: cluster 2024-09-13T00:44:46.476909+0000 mgr.y (mgr.25304) 247 : cluster [DBG] pgmap v476: 297 pgs: 29 unknown, 268 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 525 KiB/s rd, 31 KiB/s wr, 625 op/s 2024-09-13T00:44:48.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:47 smithi077 bash[83127]: cluster 2024-09-13T00:44:46.940735+0000 mon.a (mon.0) 751 : cluster [DBG] osdmap e590: 8 total, 8 up, 8 in 2024-09-13T00:44:48.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:47 smithi077 bash[83127]: audit 2024-09-13T00:44:47.727936+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:48.326 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph orch upgrade resume' 2024-09-13T00:44:48.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:47 smithi088 bash[68960]: cluster 2024-09-13T00:44:46.476909+0000 mgr.y (mgr.25304) 247 : cluster [DBG] pgmap v476: 297 pgs: 29 unknown, 268 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 525 KiB/s rd, 31 KiB/s wr, 625 op/s 2024-09-13T00:44:48.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:47 smithi088 bash[68960]: cluster 2024-09-13T00:44:46.940735+0000 mon.a (mon.0) 751 : cluster [DBG] osdmap e590: 8 total, 8 up, 8 in 2024-09-13T00:44:48.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:47 smithi088 bash[68960]: audit 2024-09-13T00:44:47.727936+0000 mon.a (mon.0) 752 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:48.596 INFO:tasks.workunit.client.0.smithi077.stdout:num images is: 1 2024-09-13T00:44:48.596 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 1 2024-09-13T00:44:48.596 INFO:tasks.workunit.client.0.smithi077.stdout:image: image34 2024-09-13T00:44:48.596 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image34 2024-09-13T00:44:48.639 INFO:tasks.workunit.client.0.smithi077.stdout:num images is: 2 2024-09-13T00:44:48.639 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 2 2024-09-13T00:44:48.640 INFO:tasks.workunit.client.0.smithi077.stdout:image: image34 2024-09-13T00:44:48.640 INFO:tasks.workunit.client.0.smithi077.stdout:image: image35 2024-09-13T00:44:48.640 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image34 2024-09-13T00:44:48.640 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image35 2024-09-13T00:44:48.738 INFO:tasks.workunit.client.0.smithi077.stdout:100.00% done 2024-09-13T00:44:48.738 INFO:tasks.workunit.client.0.smithi077.stdout:num images is: 3 2024-09-13T00:44:48.738 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 3 2024-09-13T00:44:48.738 INFO:tasks.workunit.client.0.smithi077.stdout:image: image34 2024-09-13T00:44:48.738 INFO:tasks.workunit.client.0.smithi077.stdout:image: image35 2024-09-13T00:44:48.738 INFO:tasks.workunit.client.0.smithi077.stdout:image: image36 2024-09-13T00:44:48.738 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image34 2024-09-13T00:44:48.739 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image35 2024-09-13T00:44:48.739 INFO:tasks.workunit.client.0.smithi077.stdout:expected = image36 2024-09-13T00:44:48.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:44:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:44:48] "GET /metrics HTTP/1.1" 200 48560 "" "Prometheus/2.43.0" 2024-09-13T00:44:48.821 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestDeepCopyPP (4579 ms) 2024-09-13T00:44:48.821 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnap 2024-09-13T00:44:48.821 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:49.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:49 smithi088 bash[68960]: cluster 2024-09-13T00:44:48.085754+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e591: 8 total, 8 up, 8 in 2024-09-13T00:44:49.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:49 smithi077 bash[80681]: cluster 2024-09-13T00:44:48.085754+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e591: 8 total, 8 up, 8 in 2024-09-13T00:44:49.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:49 smithi077 bash[83127]: cluster 2024-09-13T00:44:48.085754+0000 mon.a (mon.0) 753 : cluster [DBG] osdmap e591: 8 total, 8 up, 8 in 2024-09-13T00:44:50.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:50 smithi088 bash[68960]: cluster 2024-09-13T00:44:48.477531+0000 mgr.y (mgr.25304) 248 : cluster [DBG] pgmap v479: 297 pgs: 29 unknown, 268 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 511 B/s wr, 10 op/s 2024-09-13T00:44:50.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:50 smithi088 bash[68960]: cluster 2024-09-13T00:44:49.095974+0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e592: 8 total, 8 up, 8 in 2024-09-13T00:44:50.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:50 smithi077 bash[80681]: cluster 2024-09-13T00:44:48.477531+0000 mgr.y (mgr.25304) 248 : cluster [DBG] pgmap v479: 297 pgs: 29 unknown, 268 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 511 B/s wr, 10 op/s 2024-09-13T00:44:50.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:50 smithi077 bash[80681]: cluster 2024-09-13T00:44:49.095974+0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e592: 8 total, 8 up, 8 in 2024-09-13T00:44:50.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:50 smithi077 bash[83127]: cluster 2024-09-13T00:44:48.477531+0000 mgr.y (mgr.25304) 248 : cluster [DBG] pgmap v479: 297 pgs: 29 unknown, 268 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.0 KiB/s rd, 511 B/s wr, 10 op/s 2024-09-13T00:44:50.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:50 smithi077 bash[83127]: cluster 2024-09-13T00:44:49.095974+0000 mon.a (mon.0) 754 : cluster [DBG] osdmap e592: 8 total, 8 up, 8 in 2024-09-13T00:44:51.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:51 smithi077 bash[80681]: cluster 2024-09-13T00:44:50.095856+0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e593: 8 total, 8 up, 8 in 2024-09-13T00:44:51.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:51 smithi077 bash[83127]: cluster 2024-09-13T00:44:50.095856+0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e593: 8 total, 8 up, 8 in 2024-09-13T00:44:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:51 smithi088 bash[68960]: cluster 2024-09-13T00:44:50.095856+0000 mon.a (mon.0) 755 : cluster [DBG] osdmap e593: 8 total, 8 up, 8 in 2024-09-13T00:44:52.177 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 1 2024-09-13T00:44:52.177 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 1 2024-09-13T00:44:52.177 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap1 2024-09-13T00:44:52.177 INFO:tasks.workunit.client.0.smithi077.stdout:found snap1 with size 2097152 2024-09-13T00:44:52.177 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 2 2024-09-13T00:44:52.177 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 2 2024-09-13T00:44:52.177 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap1 2024-09-13T00:44:52.177 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap2 2024-09-13T00:44:52.177 INFO:tasks.workunit.client.0.smithi077.stdout:found snap1 with size 2097152 2024-09-13T00:44:52.177 INFO:tasks.workunit.client.0.smithi077.stdout:found snap2 with size 4194304 2024-09-13T00:44:52.177 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 1 2024-09-13T00:44:52.177 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 1 2024-09-13T00:44:52.178 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap2 2024-09-13T00:44:52.178 INFO:tasks.workunit.client.0.smithi077.stdout:found snap2 with size 4194304 2024-09-13T00:44:52.178 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 0 2024-09-13T00:44:52.178 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 0 2024-09-13T00:44:52.178 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnap (3356 ms) 2024-09-13T00:44:52.178 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestGetSnapShotTimeStamp 2024-09-13T00:44:52.178 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:52.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:52 smithi077 bash[80681]: cluster 2024-09-13T00:44:50.478351+0000 mgr.y (mgr.25304) 249 : cluster [DBG] pgmap v482: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 364 KiB/s rd, 24 KiB/s wr, 432 op/s 2024-09-13T00:44:52.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:52 smithi077 bash[80681]: cluster 2024-09-13T00:44:51.116983+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e594: 8 total, 8 up, 8 in 2024-09-13T00:44:52.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:52 smithi077 bash[83127]: cluster 2024-09-13T00:44:50.478351+0000 mgr.y (mgr.25304) 249 : cluster [DBG] pgmap v482: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 364 KiB/s rd, 24 KiB/s wr, 432 op/s 2024-09-13T00:44:52.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:52 smithi077 bash[83127]: cluster 2024-09-13T00:44:51.116983+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e594: 8 total, 8 up, 8 in 2024-09-13T00:44:52.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:52 smithi088 bash[68960]: cluster 2024-09-13T00:44:50.478351+0000 mgr.y (mgr.25304) 249 : cluster [DBG] pgmap v482: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 364 KiB/s rd, 24 KiB/s wr, 432 op/s 2024-09-13T00:44:52.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:52 smithi088 bash[68960]: cluster 2024-09-13T00:44:51.116983+0000 mon.a (mon.0) 756 : cluster [DBG] osdmap e594: 8 total, 8 up, 8 in 2024-09-13T00:44:52.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:44:52 smithi088 bash[61791]: ts=2024-09-13T00:44:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:44:53.138 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:44:53.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:53 smithi077 bash[80681]: cluster 2024-09-13T00:44:52.130994+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e595: 8 total, 8 up, 8 in 2024-09-13T00:44:53.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:53 smithi077 bash[83127]: cluster 2024-09-13T00:44:52.130994+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e595: 8 total, 8 up, 8 in 2024-09-13T00:44:53.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:53 smithi088 bash[68960]: cluster 2024-09-13T00:44:52.130994+0000 mon.a (mon.0) 757 : cluster [DBG] osdmap e595: 8 total, 8 up, 8 in 2024-09-13T00:44:54.193 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestGetSnapShotTimeStamp (2017 ms) 2024-09-13T00:44:54.193 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnapPP 2024-09-13T00:44:54.194 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:54 smithi077 bash[80681]: cluster 2024-09-13T00:44:52.479194+0000 mgr.y (mgr.25304) 250 : cluster [DBG] pgmap v485: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 364 KiB/s rd, 24 KiB/s wr, 434 op/s 2024-09-13T00:44:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:54 smithi077 bash[80681]: cluster 2024-09-13T00:44:53.142160+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e596: 8 total, 8 up, 8 in 2024-09-13T00:44:54.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:54 smithi077 bash[83127]: cluster 2024-09-13T00:44:52.479194+0000 mgr.y (mgr.25304) 250 : cluster [DBG] pgmap v485: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 364 KiB/s rd, 24 KiB/s wr, 434 op/s 2024-09-13T00:44:54.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:54 smithi077 bash[83127]: cluster 2024-09-13T00:44:53.142160+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e596: 8 total, 8 up, 8 in 2024-09-13T00:44:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:54 smithi088 bash[68960]: cluster 2024-09-13T00:44:52.479194+0000 mgr.y (mgr.25304) 250 : cluster [DBG] pgmap v485: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 364 KiB/s rd, 24 KiB/s wr, 434 op/s 2024-09-13T00:44:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:54 smithi088 bash[68960]: cluster 2024-09-13T00:44:53.142160+0000 mon.a (mon.0) 758 : cluster [DBG] osdmap e596: 8 total, 8 up, 8 in 2024-09-13T00:44:55.189 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 1 2024-09-13T00:44:55.190 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 1 2024-09-13T00:44:55.190 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap1 2024-09-13T00:44:55.190 INFO:tasks.workunit.client.0.smithi077.stdout:found snap1 with size 2097152 2024-09-13T00:44:55.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:55 smithi077 bash[80681]: cluster 2024-09-13T00:44:54.149107+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e597: 8 total, 8 up, 8 in 2024-09-13T00:44:55.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:55 smithi077 bash[83127]: cluster 2024-09-13T00:44:54.149107+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e597: 8 total, 8 up, 8 in 2024-09-13T00:44:55.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:55 smithi088 bash[68960]: cluster 2024-09-13T00:44:54.149107+0000 mon.a (mon.0) 759 : cluster [DBG] osdmap e597: 8 total, 8 up, 8 in 2024-09-13T00:44:55.973 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 2 2024-09-13T00:44:55.973 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 2 2024-09-13T00:44:55.973 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap1 2024-09-13T00:44:55.974 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap2 2024-09-13T00:44:55.974 INFO:tasks.workunit.client.0.smithi077.stdout:found snap1 with size 2097152 2024-09-13T00:44:55.974 INFO:tasks.workunit.client.0.smithi077.stdout:found snap2 with size 4194304 2024-09-13T00:44:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:56 smithi077 bash[80681]: cluster 2024-09-13T00:44:54.481365+0000 mgr.y (mgr.25304) 251 : cluster [DBG] pgmap v488: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 4.0 KiB/s wr, 42 op/s 2024-09-13T00:44:56.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:56 smithi077 bash[80681]: cluster 2024-09-13T00:44:55.170120+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e598: 8 total, 8 up, 8 in 2024-09-13T00:44:56.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:56 smithi077 bash[80681]: cluster 2024-09-13T00:44:55.942320+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e599: 8 total, 8 up, 8 in 2024-09-13T00:44:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:56 smithi077 bash[83127]: cluster 2024-09-13T00:44:54.481365+0000 mgr.y (mgr.25304) 251 : cluster [DBG] pgmap v488: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 4.0 KiB/s wr, 42 op/s 2024-09-13T00:44:56.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:56 smithi077 bash[83127]: cluster 2024-09-13T00:44:55.170120+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e598: 8 total, 8 up, 8 in 2024-09-13T00:44:56.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:56 smithi077 bash[83127]: cluster 2024-09-13T00:44:55.942320+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e599: 8 total, 8 up, 8 in 2024-09-13T00:44:56.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:56 smithi088 bash[68960]: cluster 2024-09-13T00:44:54.481365+0000 mgr.y (mgr.25304) 251 : cluster [DBG] pgmap v488: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 4.0 KiB/s wr, 42 op/s 2024-09-13T00:44:56.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:56 smithi088 bash[68960]: cluster 2024-09-13T00:44:55.170120+0000 mon.a (mon.0) 760 : cluster [DBG] osdmap e598: 8 total, 8 up, 8 in 2024-09-13T00:44:56.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:56 smithi088 bash[68960]: cluster 2024-09-13T00:44:55.942320+0000 mon.a (mon.0) 761 : cluster [DBG] osdmap e599: 8 total, 8 up, 8 in 2024-09-13T00:44:56.970 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 1 2024-09-13T00:44:56.970 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 1 2024-09-13T00:44:56.970 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap2 2024-09-13T00:44:56.970 INFO:tasks.workunit.client.0.smithi077.stdout:found snap2 with size 4194304 2024-09-13T00:44:57.733 INFO:teuthology.orchestra.run.smithi077.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:44:57.986 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 0 2024-09-13T00:44:57.987 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 0 2024-09-13T00:44:57.999 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnapPP (3805 ms) 2024-09-13T00:44:57.999 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestGetNameIdSnapPP 2024-09-13T00:44:57.999 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:44:58.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:57 smithi077 bash[83127]: cluster 2024-09-13T00:44:56.482528+0000 mgr.y (mgr.25304) 252 : cluster [DBG] pgmap v491: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 6.0 KiB/s wr, 71 op/s 2024-09-13T00:44:58.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:57 smithi077 bash[83127]: cluster 2024-09-13T00:44:56.943652+0000 mon.a (mon.0) 762 : cluster [DBG] osdmap e600: 8 total, 8 up, 8 in 2024-09-13T00:44:58.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:57 smithi077 bash[83127]: audit 2024-09-13T00:44:57.735644+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:58.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:57 smithi077 bash[83127]: audit 2024-09-13T00:44:57.736725+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:44:58.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:57 smithi077 bash[83127]: audit 2024-09-13T00:44:57.762115+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:58.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:57 smithi077 bash[80681]: cluster 2024-09-13T00:44:56.482528+0000 mgr.y (mgr.25304) 252 : cluster [DBG] pgmap v491: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 6.0 KiB/s wr, 71 op/s 2024-09-13T00:44:58.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:57 smithi077 bash[80681]: cluster 2024-09-13T00:44:56.943652+0000 mon.a (mon.0) 762 : cluster [DBG] osdmap e600: 8 total, 8 up, 8 in 2024-09-13T00:44:58.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:57 smithi077 bash[80681]: audit 2024-09-13T00:44:57.735644+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:58.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:57 smithi077 bash[80681]: audit 2024-09-13T00:44:57.736725+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:44:58.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:57 smithi077 bash[80681]: audit 2024-09-13T00:44:57.762115+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:58.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:57 smithi088 bash[68960]: cluster 2024-09-13T00:44:56.482528+0000 mgr.y (mgr.25304) 252 : cluster [DBG] pgmap v491: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 50 KiB/s rd, 6.0 KiB/s wr, 71 op/s 2024-09-13T00:44:58.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:57 smithi088 bash[68960]: cluster 2024-09-13T00:44:56.943652+0000 mon.a (mon.0) 762 : cluster [DBG] osdmap e600: 8 total, 8 up, 8 in 2024-09-13T00:44:58.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:57 smithi088 bash[68960]: audit 2024-09-13T00:44:57.735644+0000 mon.a (mon.0) 763 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:58.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:57 smithi088 bash[68960]: audit 2024-09-13T00:44:57.736725+0000 mon.a (mon.0) 764 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:44:58.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:57 smithi088 bash[68960]: audit 2024-09-13T00:44:57.762115+0000 mon.a (mon.0) 765 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:44:58.654 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'echo wait for half of osds to upgrade' 2024-09-13T00:44:58.967 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:44:58] "GET /metrics HTTP/1.1" 200 50983 "" "Prometheus/2.43.0" 2024-09-13T00:44:59.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[80681]: audit 2024-09-13T00:44:57.728435+0000 mgr.y (mgr.25304) 253 : audit [DBG] from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:44:59.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[80681]: cephadm 2024-09-13T00:44:57.728958+0000 mgr.y (mgr.25304) 254 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:44:59.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[80681]: cluster 2024-09-13T00:44:57.947289+0000 mon.a (mon.0) 766 : cluster [DBG] osdmap e601: 8 total, 8 up, 8 in 2024-09-13T00:44:59.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[80681]: audit 2024-09-13T00:44:58.388443+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.0", "id": [7, 5]}]: dispatch 2024-09-13T00:44:59.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[80681]: audit 2024-09-13T00:44:58.388916+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.8", "id": [7, 0]}]: dispatch 2024-09-13T00:44:59.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[80681]: audit 2024-09-13T00:44:58.389310+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 4]}]: dispatch 2024-09-13T00:44:59.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[80681]: audit 2024-09-13T00:44:58.389705+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 5]}]: dispatch 2024-09-13T00:44:59.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[80681]: audit 2024-09-13T00:44:58.390093+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]}]: dispatch 2024-09-13T00:44:59.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[80681]: audit 2024-09-13T00:44:58.390442+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]}]: dispatch 2024-09-13T00:44:59.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[80681]: audit 2024-09-13T00:44:58.390829+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]}]: dispatch 2024-09-13T00:44:59.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[80681]: audit 2024-09-13T00:44:58.417097+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:59.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[83127]: audit 2024-09-13T00:44:57.728435+0000 mgr.y (mgr.25304) 253 : audit [DBG] from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:44:59.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[83127]: cephadm 2024-09-13T00:44:57.728958+0000 mgr.y (mgr.25304) 254 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:44:59.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[83127]: cluster 2024-09-13T00:44:57.947289+0000 mon.a (mon.0) 766 : cluster [DBG] osdmap e601: 8 total, 8 up, 8 in 2024-09-13T00:44:59.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[83127]: audit 2024-09-13T00:44:58.388443+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.0", "id": [7, 5]}]: dispatch 2024-09-13T00:44:59.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[83127]: audit 2024-09-13T00:44:58.388916+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.8", "id": [7, 0]}]: dispatch 2024-09-13T00:44:59.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[83127]: audit 2024-09-13T00:44:58.389310+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 4]}]: dispatch 2024-09-13T00:44:59.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[83127]: audit 2024-09-13T00:44:58.389705+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 5]}]: dispatch 2024-09-13T00:44:59.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[83127]: audit 2024-09-13T00:44:58.390093+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]}]: dispatch 2024-09-13T00:44:59.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[83127]: audit 2024-09-13T00:44:58.390442+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]}]: dispatch 2024-09-13T00:44:59.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[83127]: audit 2024-09-13T00:44:58.390829+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]}]: dispatch 2024-09-13T00:44:59.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:44:58 smithi077 bash[83127]: audit 2024-09-13T00:44:58.417097+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:44:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:58 smithi088 bash[68960]: audit 2024-09-13T00:44:57.728435+0000 mgr.y (mgr.25304) 253 : audit [DBG] from='client.34356 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:44:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:58 smithi088 bash[68960]: cephadm 2024-09-13T00:44:57.728958+0000 mgr.y (mgr.25304) 254 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:44:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:58 smithi088 bash[68960]: cluster 2024-09-13T00:44:57.947289+0000 mon.a (mon.0) 766 : cluster [DBG] osdmap e601: 8 total, 8 up, 8 in 2024-09-13T00:44:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:58 smithi088 bash[68960]: audit 2024-09-13T00:44:58.388443+0000 mon.a (mon.0) 767 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.0", "id": [7, 5]}]: dispatch 2024-09-13T00:44:59.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:58 smithi088 bash[68960]: audit 2024-09-13T00:44:58.388916+0000 mon.a (mon.0) 768 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.8", "id": [7, 0]}]: dispatch 2024-09-13T00:44:59.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:58 smithi088 bash[68960]: audit 2024-09-13T00:44:58.389310+0000 mon.a (mon.0) 769 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 4]}]: dispatch 2024-09-13T00:44:59.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:58 smithi088 bash[68960]: audit 2024-09-13T00:44:58.389705+0000 mon.a (mon.0) 770 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 5]}]: dispatch 2024-09-13T00:44:59.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:58 smithi088 bash[68960]: audit 2024-09-13T00:44:58.390093+0000 mon.a (mon.0) 771 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]}]: dispatch 2024-09-13T00:44:59.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:58 smithi088 bash[68960]: audit 2024-09-13T00:44:58.390442+0000 mon.a (mon.0) 772 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]}]: dispatch 2024-09-13T00:44:59.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:58 smithi088 bash[68960]: audit 2024-09-13T00:44:58.390829+0000 mon.a (mon.0) 773 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]}]: dispatch 2024-09-13T00:44:59.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:58 smithi088 bash[68960]: audit 2024-09-13T00:44:58.417097+0000 mon.a (mon.0) 774 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:59 smithi077 bash[80681]: cluster 2024-09-13T00:44:58.483293+0000 mgr.y (mgr.25304) 255 : cluster [DBG] pgmap v494: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 2.2 KiB/s wr, 30 op/s 2024-09-13T00:45:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:59 smithi077 bash[80681]: audit 2024-09-13T00:44:58.979524+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.0", "id": [7, 5]}]': finished 2024-09-13T00:45:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:59 smithi077 bash[80681]: audit 2024-09-13T00:44:58.979682+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.8", "id": [7, 0]}]': finished 2024-09-13T00:45:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:59 smithi077 bash[80681]: audit 2024-09-13T00:44:58.979797+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 4]}]': finished 2024-09-13T00:45:00.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:59 smithi077 bash[80681]: audit 2024-09-13T00:44:58.979906+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 5]}]': finished 2024-09-13T00:45:00.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:59 smithi077 bash[80681]: audit 2024-09-13T00:44:58.980028+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]}]': finished 2024-09-13T00:45:00.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:59 smithi077 bash[80681]: audit 2024-09-13T00:44:58.980181+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]}]': finished 2024-09-13T00:45:00.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:59 smithi077 bash[80681]: audit 2024-09-13T00:44:58.980303+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]}]': finished 2024-09-13T00:45:00.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:44:59 smithi077 bash[80681]: cluster 2024-09-13T00:44:58.980425+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e602: 8 total, 8 up, 8 in 2024-09-13T00:45:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:00 smithi077 bash[83127]: cluster 2024-09-13T00:44:58.483293+0000 mgr.y (mgr.25304) 255 : cluster [DBG] pgmap v494: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 2.2 KiB/s wr, 30 op/s 2024-09-13T00:45:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:00 smithi077 bash[83127]: audit 2024-09-13T00:44:58.979524+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.0", "id": [7, 5]}]': finished 2024-09-13T00:45:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:00 smithi077 bash[83127]: audit 2024-09-13T00:44:58.979682+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.8", "id": [7, 0]}]': finished 2024-09-13T00:45:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:00 smithi077 bash[83127]: audit 2024-09-13T00:44:58.979797+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 4]}]': finished 2024-09-13T00:45:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:00 smithi077 bash[83127]: audit 2024-09-13T00:44:58.979906+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 5]}]': finished 2024-09-13T00:45:00.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:00 smithi077 bash[83127]: audit 2024-09-13T00:44:58.980028+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]}]': finished 2024-09-13T00:45:00.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:00 smithi077 bash[83127]: audit 2024-09-13T00:44:58.980181+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]}]': finished 2024-09-13T00:45:00.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:00 smithi077 bash[83127]: audit 2024-09-13T00:44:58.980303+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]}]': finished 2024-09-13T00:45:00.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:00 smithi077 bash[83127]: cluster 2024-09-13T00:44:58.980425+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e602: 8 total, 8 up, 8 in 2024-09-13T00:45:00.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:59 smithi088 bash[68960]: cluster 2024-09-13T00:44:58.483293+0000 mgr.y (mgr.25304) 255 : cluster [DBG] pgmap v494: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 2.2 KiB/s wr, 30 op/s 2024-09-13T00:45:00.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:59 smithi088 bash[68960]: audit 2024-09-13T00:44:58.979524+0000 mon.a (mon.0) 775 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.0", "id": [7, 5]}]': finished 2024-09-13T00:45:00.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:59 smithi088 bash[68960]: audit 2024-09-13T00:44:58.979682+0000 mon.a (mon.0) 776 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.8", "id": [7, 0]}]': finished 2024-09-13T00:45:00.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:59 smithi088 bash[68960]: audit 2024-09-13T00:44:58.979797+0000 mon.a (mon.0) 777 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.a", "id": [7, 4]}]': finished 2024-09-13T00:45:00.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:59 smithi088 bash[68960]: audit 2024-09-13T00:44:58.979906+0000 mon.a (mon.0) 778 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.18", "id": [7, 5]}]': finished 2024-09-13T00:45:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:59 smithi088 bash[68960]: audit 2024-09-13T00:44:58.980028+0000 mon.a (mon.0) 779 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]}]': finished 2024-09-13T00:45:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:59 smithi088 bash[68960]: audit 2024-09-13T00:44:58.980181+0000 mon.a (mon.0) 780 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]}]': finished 2024-09-13T00:45:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:59 smithi088 bash[68960]: audit 2024-09-13T00:44:58.980303+0000 mon.a (mon.0) 781 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.15", "id": [7, 0]}]': finished 2024-09-13T00:45:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:44:59 smithi088 bash[68960]: cluster 2024-09-13T00:44:58.980425+0000 mon.a (mon.0) 782 : cluster [DBG] osdmap e602: 8 total, 8 up, 8 in 2024-09-13T00:45:01.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:01 smithi077 bash[80681]: cluster 2024-09-13T00:44:59.986753+0000 mon.a (mon.0) 783 : cluster [DBG] osdmap e603: 8 total, 8 up, 8 in 2024-09-13T00:45:01.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:01 smithi077 bash[80681]: cluster 2024-09-13T00:45:00.943295+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e604: 8 total, 8 up, 8 in 2024-09-13T00:45:01.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:01 smithi077 bash[83127]: cluster 2024-09-13T00:44:59.986753+0000 mon.a (mon.0) 783 : cluster [DBG] osdmap e603: 8 total, 8 up, 8 in 2024-09-13T00:45:01.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:01 smithi077 bash[83127]: cluster 2024-09-13T00:45:00.943295+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e604: 8 total, 8 up, 8 in 2024-09-13T00:45:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:01 smithi088 bash[68960]: cluster 2024-09-13T00:44:59.986753+0000 mon.a (mon.0) 783 : cluster [DBG] osdmap e603: 8 total, 8 up, 8 in 2024-09-13T00:45:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:01 smithi088 bash[68960]: cluster 2024-09-13T00:45:00.943295+0000 mon.a (mon.0) 784 : cluster [DBG] osdmap e604: 8 total, 8 up, 8 in 2024-09-13T00:45:02.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:02 smithi077 bash[80681]: cluster 2024-09-13T00:45:00.484167+0000 mgr.y (mgr.25304) 256 : cluster [DBG] pgmap v497: 297 pgs: 7 peering, 290 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 5.5 KiB/s wr, 49 op/s 2024-09-13T00:45:02.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:02 smithi077 bash[83127]: cluster 2024-09-13T00:45:00.484167+0000 mgr.y (mgr.25304) 256 : cluster [DBG] pgmap v497: 297 pgs: 7 peering, 290 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 5.5 KiB/s wr, 49 op/s 2024-09-13T00:45:02.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:02 smithi088 bash[68960]: cluster 2024-09-13T00:45:00.484167+0000 mgr.y (mgr.25304) 256 : cluster [DBG] pgmap v497: 297 pgs: 7 peering, 290 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 35 KiB/s rd, 5.5 KiB/s wr, 49 op/s 2024-09-13T00:45:02.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:45:02 smithi088 bash[61791]: ts=2024-09-13T00:45:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:45:02.613 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:45:03.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:03 smithi077 bash[80681]: cluster 2024-09-13T00:45:02.008061+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2024-09-13T00:45:03.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:03 smithi077 bash[83127]: cluster 2024-09-13T00:45:02.008061+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2024-09-13T00:45:03.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:03 smithi088 bash[68960]: cluster 2024-09-13T00:45:02.008061+0000 mon.a (mon.0) 785 : cluster [DBG] osdmap e605: 8 total, 8 up, 8 in 2024-09-13T00:45:04.063 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 0 2024-09-13T00:45:04.063 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 0 2024-09-13T00:45:04.086 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestGetNameIdSnapPP (6088 ms) 2024-09-13T00:45:04.086 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCreateLsRenameSnapPP 2024-09-13T00:45:04.086 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:45:04.180 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:03 smithi077 bash[73137]: debug 2024-09-13T00:45:03.909+0000 7ffa94da1640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-13T00:45:04.180 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[80681]: cluster 2024-09-13T00:45:02.485025+0000 mgr.y (mgr.25304) 257 : cluster [DBG] pgmap v500: 297 pgs: 7 peering, 290 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 5.7 KiB/s wr, 52 op/s 2024-09-13T00:45:04.180 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[80681]: cluster 2024-09-13T00:45:03.027753+0000 mon.a (mon.0) 786 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2024-09-13T00:45:04.180 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[80681]: audit 2024-09-13T00:45:03.311466+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:04.180 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[80681]: audit 2024-09-13T00:45:03.317675+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:04.180 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[80681]: audit 2024-09-13T00:45:03.782180+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:45:04.181 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[80681]: audit 2024-09-13T00:45:03.783647+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:45:04.181 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[80681]: audit 2024-09-13T00:45:03.792921+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:04.181 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[80681]: audit 2024-09-13T00:45:03.900588+0000 mon.a (mon.0) 792 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:45:04.181 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[80681]: audit 2024-09-13T00:45:03.904341+0000 mon.a (mon.0) 793 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:04.181 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[80681]: audit 2024-09-13T00:45:03.906511+0000 mon.a (mon.0) 794 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:04.181 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[80681]: audit 2024-09-13T00:45:03.908153+0000 mon.a (mon.0) 795 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:04.181 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[80681]: audit 2024-09-13T00:45:03.910085+0000 mon.a (mon.0) 796 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T00:45:04.181 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[83127]: cluster 2024-09-13T00:45:02.485025+0000 mgr.y (mgr.25304) 257 : cluster [DBG] pgmap v500: 297 pgs: 7 peering, 290 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 5.7 KiB/s wr, 52 op/s 2024-09-13T00:45:04.181 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[83127]: cluster 2024-09-13T00:45:03.027753+0000 mon.a (mon.0) 786 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2024-09-13T00:45:04.181 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[83127]: audit 2024-09-13T00:45:03.311466+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:04.181 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[83127]: audit 2024-09-13T00:45:03.317675+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:04.182 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[83127]: audit 2024-09-13T00:45:03.782180+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:45:04.182 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[83127]: audit 2024-09-13T00:45:03.783647+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:45:04.182 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[83127]: audit 2024-09-13T00:45:03.792921+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:04.182 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[83127]: audit 2024-09-13T00:45:03.900588+0000 mon.a (mon.0) 792 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:45:04.182 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[83127]: audit 2024-09-13T00:45:03.904341+0000 mon.a (mon.0) 793 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:04.182 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[83127]: audit 2024-09-13T00:45:03.906511+0000 mon.a (mon.0) 794 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:04.182 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[83127]: audit 2024-09-13T00:45:03.908153+0000 mon.a (mon.0) 795 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:04.182 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:04 smithi077 bash[83127]: audit 2024-09-13T00:45:03.910085+0000 mon.a (mon.0) 796 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T00:45:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:04 smithi088 bash[68960]: cluster 2024-09-13T00:45:02.485025+0000 mgr.y (mgr.25304) 257 : cluster [DBG] pgmap v500: 297 pgs: 7 peering, 290 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 36 KiB/s rd, 5.7 KiB/s wr, 52 op/s 2024-09-13T00:45:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:04 smithi088 bash[68960]: cluster 2024-09-13T00:45:03.027753+0000 mon.a (mon.0) 786 : cluster [DBG] osdmap e606: 8 total, 8 up, 8 in 2024-09-13T00:45:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:04 smithi088 bash[68960]: audit 2024-09-13T00:45:03.311466+0000 mon.a (mon.0) 787 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:04 smithi088 bash[68960]: audit 2024-09-13T00:45:03.317675+0000 mon.a (mon.0) 788 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:04 smithi088 bash[68960]: audit 2024-09-13T00:45:03.782180+0000 mon.a (mon.0) 789 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:45:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:04 smithi088 bash[68960]: audit 2024-09-13T00:45:03.783647+0000 mon.a (mon.0) 790 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:45:04.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:04 smithi088 bash[68960]: audit 2024-09-13T00:45:03.792921+0000 mon.a (mon.0) 791 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:04.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:04 smithi088 bash[68960]: audit 2024-09-13T00:45:03.900588+0000 mon.a (mon.0) 792 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:45:04.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:04 smithi088 bash[68960]: audit 2024-09-13T00:45:03.904341+0000 mon.a (mon.0) 793 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:04.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:04 smithi088 bash[68960]: audit 2024-09-13T00:45:03.906511+0000 mon.a (mon.0) 794 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:04.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:04 smithi088 bash[68960]: audit 2024-09-13T00:45:03.908153+0000 mon.a (mon.0) 795 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:04.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:04 smithi088 bash[68960]: audit 2024-09-13T00:45:03.910085+0000 mon.a (mon.0) 796 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T00:45:05.082 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 1 2024-09-13T00:45:05.082 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 1 2024-09-13T00:45:05.082 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap1 2024-09-13T00:45:05.082 INFO:tasks.workunit.client.0.smithi077.stdout:found snap1 with size 2097152 2024-09-13T00:45:05.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:05 smithi088 bash[68960]: audit 2024-09-13T00:45:03.910377+0000 mgr.y (mgr.25304) 258 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T00:45:05.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:05 smithi088 bash[68960]: cephadm 2024-09-13T00:45:03.911841+0000 mgr.y (mgr.25304) 259 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-13T00:45:05.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:05 smithi088 bash[68960]: cluster 2024-09-13T00:45:04.041790+0000 mon.a (mon.0) 797 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2024-09-13T00:45:05.360 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:05 smithi077 bash[83127]: audit 2024-09-13T00:45:03.910377+0000 mgr.y (mgr.25304) 258 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T00:45:05.360 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:05 smithi077 bash[83127]: cephadm 2024-09-13T00:45:03.911841+0000 mgr.y (mgr.25304) 259 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-13T00:45:05.361 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:05 smithi077 bash[83127]: cluster 2024-09-13T00:45:04.041790+0000 mon.a (mon.0) 797 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2024-09-13T00:45:05.361 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:05 smithi077 bash[80681]: audit 2024-09-13T00:45:03.910377+0000 mgr.y (mgr.25304) 258 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T00:45:05.361 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:05 smithi077 bash[80681]: cephadm 2024-09-13T00:45:03.911841+0000 mgr.y (mgr.25304) 259 : cephadm [INF] Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-13T00:45:05.361 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:05 smithi077 bash[80681]: cluster 2024-09-13T00:45:04.041790+0000 mon.a (mon.0) 797 : cluster [DBG] osdmap e607: 8 total, 8 up, 8 in 2024-09-13T00:45:05.978 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 2 2024-09-13T00:45:05.978 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 2 2024-09-13T00:45:05.978 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap1 2024-09-13T00:45:05.979 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap2 2024-09-13T00:45:05.979 INFO:tasks.workunit.client.0.smithi077.stdout:found snap1 with size 2097152 2024-09-13T00:45:05.979 INFO:tasks.workunit.client.0.smithi077.stdout:found snap2 with size 4194304 2024-09-13T00:45:05.990 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 2 2024-09-13T00:45:05.990 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 2 2024-09-13T00:45:05.990 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap1-rename 2024-09-13T00:45:05.990 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap2 2024-09-13T00:45:05.990 INFO:tasks.workunit.client.0.smithi077.stdout:found snap1-rename with size 2097152 2024-09-13T00:45:05.990 INFO:tasks.workunit.client.0.smithi077.stdout:found snap2 with size 4194304 2024-09-13T00:45:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:06 smithi088 bash[68960]: cluster 2024-09-13T00:45:04.488098+0000 mgr.y (mgr.25304) 260 : cluster [DBG] pgmap v503: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 69 KiB/s rd, 5.7 KiB/s wr, 95 op/s; 12 B/s, 0 objects/s recovering 2024-09-13T00:45:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:06 smithi088 bash[68960]: cluster 2024-09-13T00:45:05.051432+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2024-09-13T00:45:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:06 smithi088 bash[68960]: cluster 2024-09-13T00:45:05.948151+0000 mon.a (mon.0) 799 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2024-09-13T00:45:06.520 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:06 smithi077 bash[80681]: cluster 2024-09-13T00:45:04.488098+0000 mgr.y (mgr.25304) 260 : cluster [DBG] pgmap v503: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 69 KiB/s rd, 5.7 KiB/s wr, 95 op/s; 12 B/s, 0 objects/s recovering 2024-09-13T00:45:06.520 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:06 smithi077 bash[80681]: cluster 2024-09-13T00:45:05.051432+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2024-09-13T00:45:06.520 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:06 smithi077 bash[80681]: cluster 2024-09-13T00:45:05.948151+0000 mon.a (mon.0) 799 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2024-09-13T00:45:06.521 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:06 smithi077 bash[83127]: cluster 2024-09-13T00:45:04.488098+0000 mgr.y (mgr.25304) 260 : cluster [DBG] pgmap v503: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 69 KiB/s rd, 5.7 KiB/s wr, 95 op/s; 12 B/s, 0 objects/s recovering 2024-09-13T00:45:06.521 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:06 smithi077 bash[83127]: cluster 2024-09-13T00:45:05.051432+0000 mon.a (mon.0) 798 : cluster [DBG] osdmap e608: 8 total, 8 up, 8 in 2024-09-13T00:45:06.521 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:06 smithi077 bash[83127]: cluster 2024-09-13T00:45:05.948151+0000 mon.a (mon.0) 799 : cluster [DBG] osdmap e609: 8 total, 8 up, 8 in 2024-09-13T00:45:06.587 INFO:teuthology.orchestra.run.smithi077.stdout:wait for half of osds to upgrade 2024-09-13T00:45:06.986 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 1 2024-09-13T00:45:06.987 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 1 2024-09-13T00:45:06.987 INFO:tasks.workunit.client.0.smithi077.stdout:snap: snap2-rename 2024-09-13T00:45:06.987 INFO:tasks.workunit.client.0.smithi077.stdout:found snap2-rename with size 4194304 2024-09-13T00:45:07.410 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-09-13T00:45:07.985 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 0 2024-09-13T00:45:07.985 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 0 2024-09-13T00:45:08.009 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCreateLsRenameSnapPP (3923 ms) 2024-09-13T00:45:08.009 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ConcurrentCreatesUnvalidatedPool 2024-09-13T00:45:08.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:07 smithi077 bash[80681]: cluster 2024-09-13T00:45:06.489323+0000 mgr.y (mgr.25304) 261 : cluster [DBG] pgmap v506: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 5.5 KiB/s wr, 92 op/s; 12 B/s, 0 objects/s recovering 2024-09-13T00:45:08.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:07 smithi077 bash[80681]: cluster 2024-09-13T00:45:06.950241+0000 mon.a (mon.0) 800 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2024-09-13T00:45:08.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:07 smithi077 bash[83127]: cluster 2024-09-13T00:45:06.489323+0000 mgr.y (mgr.25304) 261 : cluster [DBG] pgmap v506: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 5.5 KiB/s wr, 92 op/s; 12 B/s, 0 objects/s recovering 2024-09-13T00:45:08.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:07 smithi077 bash[83127]: cluster 2024-09-13T00:45:06.950241+0000 mon.a (mon.0) 800 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2024-09-13T00:45:08.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:07 smithi088 bash[68960]: cluster 2024-09-13T00:45:06.489323+0000 mgr.y (mgr.25304) 261 : cluster [DBG] pgmap v506: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 5.5 KiB/s wr, 92 op/s; 12 B/s, 0 objects/s recovering 2024-09-13T00:45:08.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:07 smithi088 bash[68960]: cluster 2024-09-13T00:45:06.950241+0000 mon.a (mon.0) 800 : cluster [DBG] osdmap e610: 8 total, 8 up, 8 in 2024-09-13T00:45:08.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:45:08] "GET /metrics HTTP/1.1" 200 50983 "" "Prometheus/2.43.0" 2024-09-13T00:45:09.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:08 smithi077 bash[80681]: cluster 2024-09-13T00:45:07.952381+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2024-09-13T00:45:09.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:08 smithi077 bash[83127]: cluster 2024-09-13T00:45:07.952381+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2024-09-13T00:45:09.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:08 smithi088 bash[68960]: cluster 2024-09-13T00:45:07.952381+0000 mon.a (mon.0) 801 : cluster [DBG] osdmap e611: 8 total, 8 up, 8 in 2024-09-13T00:45:09.993 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:45:09.993 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:45:09.993 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:45:09.993 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:45:10.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:09 smithi077 bash[80681]: cluster 2024-09-13T00:45:08.489851+0000 mgr.y (mgr.25304) 262 : cluster [DBG] pgmap v509: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:45:10.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:09 smithi077 bash[80681]: cluster 2024-09-13T00:45:08.978101+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2024-09-13T00:45:10.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:09 smithi077 bash[80681]: audit 2024-09-13T00:45:08.982512+0000 mon.c (mon.2) 28 : audit [INF] from='client.? 172.21.15.77:0/415504094' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:10.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:09 smithi077 bash[80681]: audit 2024-09-13T00:45:08.983341+0000 mon.a (mon.0) 803 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:10.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:09 smithi077 bash[83127]: cluster 2024-09-13T00:45:08.489851+0000 mgr.y (mgr.25304) 262 : cluster [DBG] pgmap v509: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:45:10.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:09 smithi077 bash[83127]: cluster 2024-09-13T00:45:08.978101+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2024-09-13T00:45:10.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:09 smithi077 bash[83127]: audit 2024-09-13T00:45:08.982512+0000 mon.c (mon.2) 28 : audit [INF] from='client.? 172.21.15.77:0/415504094' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:10.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:09 smithi077 bash[83127]: audit 2024-09-13T00:45:08.983341+0000 mon.a (mon.0) 803 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:10.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:09 smithi088 bash[68960]: cluster 2024-09-13T00:45:08.489851+0000 mgr.y (mgr.25304) 262 : cluster [DBG] pgmap v509: 297 pgs: 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:45:10.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:10 smithi088 bash[68960]: cluster 2024-09-13T00:45:08.978101+0000 mon.a (mon.0) 802 : cluster [DBG] osdmap e612: 8 total, 8 up, 8 in 2024-09-13T00:45:10.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:10 smithi088 bash[68960]: audit 2024-09-13T00:45:08.982512+0000 mon.c (mon.2) 28 : audit [INF] from='client.? 172.21.15.77:0/415504094' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:10.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:10 smithi088 bash[68960]: audit 2024-09-13T00:45:08.983341+0000 mon.a (mon.0) 803 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:11.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:11 smithi077 bash[80681]: audit 2024-09-13T00:45:09.983264+0000 mon.a (mon.0) 804 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:45:11.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:11 smithi077 bash[80681]: cluster 2024-09-13T00:45:09.983853+0000 mon.a (mon.0) 805 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2024-09-13T00:45:11.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:11 smithi077 bash[83127]: audit 2024-09-13T00:45:09.983264+0000 mon.a (mon.0) 804 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:45:11.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:11 smithi077 bash[83127]: cluster 2024-09-13T00:45:09.983853+0000 mon.a (mon.0) 805 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2024-09-13T00:45:11.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:11 smithi088 bash[68960]: audit 2024-09-13T00:45:09.983264+0000 mon.a (mon.0) 804 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:45:11.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:11 smithi088 bash[68960]: cluster 2024-09-13T00:45:09.983853+0000 mon.a (mon.0) 805 : cluster [DBG] osdmap e613: 8 total, 8 up, 8 in 2024-09-13T00:45:12.201 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:45:12.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:12 smithi077 bash[80681]: cluster 2024-09-13T00:45:10.490724+0000 mgr.y (mgr.25304) 263 : cluster [DBG] pgmap v512: 329 pgs: 24 creating+peering, 8 unknown, 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s rd, 4.7 KiB/s wr, 65 op/s 2024-09-13T00:45:12.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:12 smithi077 bash[80681]: cluster 2024-09-13T00:45:11.008798+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2024-09-13T00:45:12.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:12 smithi077 bash[83127]: cluster 2024-09-13T00:45:10.490724+0000 mgr.y (mgr.25304) 263 : cluster [DBG] pgmap v512: 329 pgs: 24 creating+peering, 8 unknown, 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s rd, 4.7 KiB/s wr, 65 op/s 2024-09-13T00:45:12.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:12 smithi077 bash[83127]: cluster 2024-09-13T00:45:11.008798+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2024-09-13T00:45:12.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:12 smithi088 bash[68960]: cluster 2024-09-13T00:45:10.490724+0000 mgr.y (mgr.25304) 263 : cluster [DBG] pgmap v512: 329 pgs: 24 creating+peering, 8 unknown, 297 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s rd, 4.7 KiB/s wr, 65 op/s 2024-09-13T00:45:12.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:12 smithi088 bash[68960]: cluster 2024-09-13T00:45:11.008798+0000 mon.a (mon.0) 806 : cluster [DBG] osdmap e614: 8 total, 8 up, 8 in 2024-09-13T00:45:12.479 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ConcurrentCreatesUnvalidatedPool (4469 ms) 2024-09-13T00:45:12.479 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.RemoveFullTry 2024-09-13T00:45:12.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:45:12 smithi088 bash[61791]: ts=2024-09-13T00:45:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:45:13.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:13 smithi088 bash[68960]: cluster 2024-09-13T00:45:12.033741+0000 mon.a (mon.0) 807 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2024-09-13T00:45:13.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:13 smithi088 bash[68960]: audit 2024-09-13T00:45:12.955913+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:13.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:13 smithi077 bash[80681]: cluster 2024-09-13T00:45:12.033741+0000 mon.a (mon.0) 807 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2024-09-13T00:45:13.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:13 smithi077 bash[80681]: audit 2024-09-13T00:45:12.955913+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:13.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:13 smithi077 bash[83127]: cluster 2024-09-13T00:45:12.033741+0000 mon.a (mon.0) 807 : cluster [DBG] osdmap e615: 8 total, 8 up, 8 in 2024-09-13T00:45:13.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:13 smithi077 bash[83127]: audit 2024-09-13T00:45:12.955913+0000 mon.a (mon.0) 808 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:14.105 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:45:14.278 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:14 smithi077 bash[73137]: [13/Sep/2024:00:45:14] ENGINE Bus STOPPING 2024-09-13T00:45:14.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:14 smithi077 bash[80681]: cluster 2024-09-13T00:45:12.491323+0000 mgr.y (mgr.25304) 264 : cluster [DBG] pgmap v515: 329 pgs: 24 creating+peering, 305 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s rd, 5.0 KiB/s wr, 65 op/s 2024-09-13T00:45:14.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:14 smithi077 bash[80681]: cluster 2024-09-13T00:45:13.044561+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e616: 8 total, 8 up, 8 in 2024-09-13T00:45:14.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:14 smithi077 bash[80681]: audit 2024-09-13T00:45:13.047135+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.77:0/735062644' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:14.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:14 smithi077 bash[80681]: audit 2024-09-13T00:45:13.048269+0000 mon.a (mon.0) 810 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:14.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:14 smithi077 bash[80681]: audit 2024-09-13T00:45:13.417428+0000 mon.a (mon.0) 811 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:14.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:14 smithi077 bash[83127]: cluster 2024-09-13T00:45:12.491323+0000 mgr.y (mgr.25304) 264 : cluster [DBG] pgmap v515: 329 pgs: 24 creating+peering, 305 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s rd, 5.0 KiB/s wr, 65 op/s 2024-09-13T00:45:14.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:14 smithi077 bash[83127]: cluster 2024-09-13T00:45:13.044561+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e616: 8 total, 8 up, 8 in 2024-09-13T00:45:14.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:14 smithi077 bash[83127]: audit 2024-09-13T00:45:13.047135+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.77:0/735062644' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:14.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:14 smithi077 bash[83127]: audit 2024-09-13T00:45:13.048269+0000 mon.a (mon.0) 810 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:14.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:14 smithi077 bash[83127]: audit 2024-09-13T00:45:13.417428+0000 mon.a (mon.0) 811 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:14.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:14 smithi088 bash[68960]: cluster 2024-09-13T00:45:12.491323+0000 mgr.y (mgr.25304) 264 : cluster [DBG] pgmap v515: 329 pgs: 24 creating+peering, 305 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 49 KiB/s rd, 5.0 KiB/s wr, 65 op/s 2024-09-13T00:45:14.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:14 smithi088 bash[68960]: cluster 2024-09-13T00:45:13.044561+0000 mon.a (mon.0) 809 : cluster [DBG] osdmap e616: 8 total, 8 up, 8 in 2024-09-13T00:45:14.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:14 smithi088 bash[68960]: audit 2024-09-13T00:45:13.047135+0000 mon.c (mon.2) 29 : audit [INF] from='client.? 172.21.15.77:0/735062644' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:14.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:14 smithi088 bash[68960]: audit 2024-09-13T00:45:13.048269+0000 mon.a (mon.0) 810 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:14.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:14 smithi088 bash[68960]: audit 2024-09-13T00:45:13.417428+0000 mon.a (mon.0) 811 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:14.641 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:14 smithi077 bash[73137]: [13/Sep/2024:00:45:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:45:14.641 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:14 smithi077 bash[73137]: [13/Sep/2024:00:45:14] ENGINE Bus STOPPED 2024-09-13T00:45:14.641 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:14 smithi077 bash[73137]: [13/Sep/2024:00:45:14] ENGINE Bus STARTING 2024-09-13T00:45:15.028 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:14 smithi077 bash[73137]: [13/Sep/2024:00:45:14] ENGINE Serving on http://:::9283 2024-09-13T00:45:15.028 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:14 smithi077 bash[73137]: [13/Sep/2024:00:45:14] ENGINE Bus STARTED 2024-09-13T00:45:15.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:15 smithi088 bash[68960]: audit 2024-09-13T00:45:14.050230+0000 mon.a (mon.0) 812 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:45:15.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:15 smithi088 bash[68960]: cluster 2024-09-13T00:45:14.050383+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e617: 8 total, 8 up, 8 in 2024-09-13T00:45:15.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:15 smithi088 bash[68960]: audit 2024-09-13T00:45:14.108787+0000 mon.a (mon.0) 814 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' 2024-09-13T00:45:15.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:15 smithi077 bash[83127]: audit 2024-09-13T00:45:14.050230+0000 mon.a (mon.0) 812 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:45:15.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:15 smithi077 bash[83127]: cluster 2024-09-13T00:45:14.050383+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e617: 8 total, 8 up, 8 in 2024-09-13T00:45:15.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:15 smithi077 bash[83127]: audit 2024-09-13T00:45:14.108787+0000 mon.a (mon.0) 814 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' 2024-09-13T00:45:15.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:15 smithi077 bash[80681]: audit 2024-09-13T00:45:14.050230+0000 mon.a (mon.0) 812 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:45:15.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:15 smithi077 bash[80681]: cluster 2024-09-13T00:45:14.050383+0000 mon.a (mon.0) 813 : cluster [DBG] osdmap e617: 8 total, 8 up, 8 in 2024-09-13T00:45:15.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:15 smithi077 bash[80681]: audit 2024-09-13T00:45:14.108787+0000 mon.a (mon.0) 814 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' 2024-09-13T00:45:16.136 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:45:16.333 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:16 smithi077 bash[80681]: cluster 2024-09-13T00:45:14.494100+0000 mgr.y (mgr.25304) 265 : cluster [DBG] pgmap v518: 361 pgs: 2 unknown, 359 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s rd, 6.7 KiB/s wr, 87 op/s 2024-09-13T00:45:16.333 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:16 smithi077 bash[80681]: cluster 2024-09-13T00:45:15.057691+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e618: 8 total, 8 up, 8 in 2024-09-13T00:45:16.333 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:16 smithi077 bash[80681]: cluster 2024-09-13T00:45:15.950118+0000 mon.a (mon.0) 816 : cluster [DBG] osdmap e619: 8 total, 8 up, 8 in 2024-09-13T00:45:16.333 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:16 smithi077 bash[83127]: cluster 2024-09-13T00:45:14.494100+0000 mgr.y (mgr.25304) 265 : cluster [DBG] pgmap v518: 361 pgs: 2 unknown, 359 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s rd, 6.7 KiB/s wr, 87 op/s 2024-09-13T00:45:16.333 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:16 smithi077 bash[83127]: cluster 2024-09-13T00:45:15.057691+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e618: 8 total, 8 up, 8 in 2024-09-13T00:45:16.334 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:16 smithi077 bash[83127]: cluster 2024-09-13T00:45:15.950118+0000 mon.a (mon.0) 816 : cluster [DBG] osdmap e619: 8 total, 8 up, 8 in 2024-09-13T00:45:16.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:16 smithi088 bash[68960]: cluster 2024-09-13T00:45:14.494100+0000 mgr.y (mgr.25304) 265 : cluster [DBG] pgmap v518: 361 pgs: 2 unknown, 359 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 56 KiB/s rd, 6.7 KiB/s wr, 87 op/s 2024-09-13T00:45:16.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:16 smithi088 bash[68960]: cluster 2024-09-13T00:45:15.057691+0000 mon.a (mon.0) 815 : cluster [DBG] osdmap e618: 8 total, 8 up, 8 in 2024-09-13T00:45:16.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:16 smithi088 bash[68960]: cluster 2024-09-13T00:45:15.950118+0000 mon.a (mon.0) 816 : cluster [DBG] osdmap e619: 8 total, 8 up, 8 in 2024-09-13T00:45:17.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:17 smithi088 bash[68960]: audit 2024-09-13T00:45:16.161186+0000 mon.a (mon.0) 817 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi077-68798-22", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:45:17.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:17 smithi088 bash[68960]: audit 2024-09-13T00:45:16.651229+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.77:0/1367894881' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:17.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:17 smithi077 bash[80681]: audit 2024-09-13T00:45:16.161186+0000 mon.a (mon.0) 817 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi077-68798-22", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:45:17.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:17 smithi077 bash[80681]: audit 2024-09-13T00:45:16.651229+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.77:0/1367894881' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:17.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:17 smithi077 bash[83127]: audit 2024-09-13T00:45:16.161186+0000 mon.a (mon.0) 817 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi077-68798-22", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:45:17.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:17 smithi077 bash[83127]: audit 2024-09-13T00:45:16.651229+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.77:0/1367894881' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:18.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:18 smithi088 bash[68960]: cluster 2024-09-13T00:45:16.495082+0000 mgr.y (mgr.25304) 266 : cluster [DBG] pgmap v521: 361 pgs: 361 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s rd, 8.2 KiB/s wr, 144 op/s 2024-09-13T00:45:18.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:18 smithi088 bash[68960]: audit 2024-09-13T00:45:17.076460+0000 mon.a (mon.0) 818 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi077-68798-22", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T00:45:18.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:18 smithi088 bash[68960]: cluster 2024-09-13T00:45:17.076607+0000 mon.a (mon.0) 819 : cluster [DBG] osdmap e620: 8 total, 8 up, 8 in 2024-09-13T00:45:18.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:18 smithi077 bash[80681]: cluster 2024-09-13T00:45:16.495082+0000 mgr.y (mgr.25304) 266 : cluster [DBG] pgmap v521: 361 pgs: 361 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s rd, 8.2 KiB/s wr, 144 op/s 2024-09-13T00:45:18.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:18 smithi077 bash[80681]: audit 2024-09-13T00:45:17.076460+0000 mon.a (mon.0) 818 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi077-68798-22", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T00:45:18.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:18 smithi077 bash[80681]: cluster 2024-09-13T00:45:17.076607+0000 mon.a (mon.0) 819 : cluster [DBG] osdmap e620: 8 total, 8 up, 8 in 2024-09-13T00:45:18.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:18 smithi077 bash[83127]: cluster 2024-09-13T00:45:16.495082+0000 mgr.y (mgr.25304) 266 : cluster [DBG] pgmap v521: 361 pgs: 361 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 100 KiB/s rd, 8.2 KiB/s wr, 144 op/s 2024-09-13T00:45:18.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:18 smithi077 bash[83127]: audit 2024-09-13T00:45:17.076460+0000 mon.a (mon.0) 818 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi077-68798-22", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T00:45:18.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:18 smithi077 bash[83127]: cluster 2024-09-13T00:45:17.076607+0000 mon.a (mon.0) 819 : cluster [DBG] osdmap e620: 8 total, 8 up, 8 in 2024-09-13T00:45:18.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:45:18] "GET /metrics HTTP/1.1" 200 55729 "" "Prometheus/2.43.0" 2024-09-13T00:45:19.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:19 smithi088 bash[68960]: audit 2024-09-13T00:45:18.927722+0000 mon.a (mon.0) 820 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T00:45:19.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:19 smithi088 bash[68960]: audit 2024-09-13T00:45:18.944809+0000 mon.a (mon.0) 821 : audit [DBG] from='client.? 172.21.15.77:0/2962080899' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:19.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:19 smithi077 bash[80681]: audit 2024-09-13T00:45:18.927722+0000 mon.a (mon.0) 820 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T00:45:19.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:19 smithi077 bash[80681]: audit 2024-09-13T00:45:18.944809+0000 mon.a (mon.0) 821 : audit [DBG] from='client.? 172.21.15.77:0/2962080899' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:19.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:19 smithi077 bash[83127]: audit 2024-09-13T00:45:18.927722+0000 mon.a (mon.0) 820 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T00:45:19.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:19 smithi077 bash[83127]: audit 2024-09-13T00:45:18.944809+0000 mon.a (mon.0) 821 : audit [DBG] from='client.? 172.21.15.77:0/2962080899' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:20.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:20 smithi088 bash[68960]: cluster 2024-09-13T00:45:18.495943+0000 mgr.y (mgr.25304) 267 : cluster [DBG] pgmap v523: 361 pgs: 361 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 90 KiB/s rd, 7.4 KiB/s wr, 129 op/s 2024-09-13T00:45:20.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:20 smithi088 bash[68960]: audit 2024-09-13T00:45:18.928142+0000 mgr.y (mgr.25304) 268 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T00:45:20.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:20 smithi088 bash[68960]: cephadm 2024-09-13T00:45:18.931367+0000 mgr.y (mgr.25304) 269 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-09-13T00:45:20.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:20 smithi077 bash[80681]: cluster 2024-09-13T00:45:18.495943+0000 mgr.y (mgr.25304) 267 : cluster [DBG] pgmap v523: 361 pgs: 361 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 90 KiB/s rd, 7.4 KiB/s wr, 129 op/s 2024-09-13T00:45:20.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:20 smithi077 bash[80681]: audit 2024-09-13T00:45:18.928142+0000 mgr.y (mgr.25304) 268 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T00:45:20.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:20 smithi077 bash[80681]: cephadm 2024-09-13T00:45:18.931367+0000 mgr.y (mgr.25304) 269 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-09-13T00:45:20.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:20 smithi077 bash[83127]: cluster 2024-09-13T00:45:18.495943+0000 mgr.y (mgr.25304) 267 : cluster [DBG] pgmap v523: 361 pgs: 361 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 90 KiB/s rd, 7.4 KiB/s wr, 129 op/s 2024-09-13T00:45:20.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:20 smithi077 bash[83127]: audit 2024-09-13T00:45:18.928142+0000 mgr.y (mgr.25304) 268 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-13T00:45:20.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:20 smithi077 bash[83127]: cephadm 2024-09-13T00:45:18.931367+0000 mgr.y (mgr.25304) 269 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-09-13T00:45:22.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:21 smithi077 bash[80681]: cluster 2024-09-13T00:45:20.498978+0000 mgr.y (mgr.25304) 270 : cluster [DBG] pgmap v524: 361 pgs: 361 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 3.5 KiB/s wr, 71 op/s 2024-09-13T00:45:22.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:21 smithi077 bash[80681]: cluster 2024-09-13T00:45:20.952403+0000 mon.a (mon.0) 822 : cluster [DBG] osdmap e621: 8 total, 8 up, 8 in 2024-09-13T00:45:22.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:21 smithi077 bash[80681]: audit 2024-09-13T00:45:21.248970+0000 mon.a (mon.0) 823 : audit [DBG] from='client.? 172.21.15.77:0/1436600673' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:22.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:21 smithi077 bash[80681]: audit 2024-09-13T00:45:21.581749+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:22.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:21 smithi077 bash[80681]: audit 2024-09-13T00:45:21.583525+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T00:45:22.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:21 smithi077 bash[80681]: audit 2024-09-13T00:45:21.585009+0000 mon.a (mon.0) 826 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:45:22.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:21 smithi077 bash[83127]: cluster 2024-09-13T00:45:20.498978+0000 mgr.y (mgr.25304) 270 : cluster [DBG] pgmap v524: 361 pgs: 361 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 3.5 KiB/s wr, 71 op/s 2024-09-13T00:45:22.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:21 smithi077 bash[83127]: cluster 2024-09-13T00:45:20.952403+0000 mon.a (mon.0) 822 : cluster [DBG] osdmap e621: 8 total, 8 up, 8 in 2024-09-13T00:45:22.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:21 smithi077 bash[83127]: audit 2024-09-13T00:45:21.248970+0000 mon.a (mon.0) 823 : audit [DBG] from='client.? 172.21.15.77:0/1436600673' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:22.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:21 smithi077 bash[83127]: audit 2024-09-13T00:45:21.581749+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:22.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:21 smithi077 bash[83127]: audit 2024-09-13T00:45:21.583525+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T00:45:22.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:21 smithi077 bash[83127]: audit 2024-09-13T00:45:21.585009+0000 mon.a (mon.0) 826 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:45:22.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:21 smithi088 bash[68960]: cluster 2024-09-13T00:45:20.498978+0000 mgr.y (mgr.25304) 270 : cluster [DBG] pgmap v524: 361 pgs: 361 active+clean; 584 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 51 KiB/s rd, 3.5 KiB/s wr, 71 op/s 2024-09-13T00:45:22.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:21 smithi088 bash[68960]: cluster 2024-09-13T00:45:20.952403+0000 mon.a (mon.0) 822 : cluster [DBG] osdmap e621: 8 total, 8 up, 8 in 2024-09-13T00:45:22.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:21 smithi088 bash[68960]: audit 2024-09-13T00:45:21.248970+0000 mon.a (mon.0) 823 : audit [DBG] from='client.? 172.21.15.77:0/1436600673' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:22.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:21 smithi088 bash[68960]: audit 2024-09-13T00:45:21.581749+0000 mon.a (mon.0) 824 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:22.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:21 smithi088 bash[68960]: audit 2024-09-13T00:45:21.583525+0000 mon.a (mon.0) 825 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-13T00:45:22.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:21 smithi088 bash[68960]: audit 2024-09-13T00:45:21.585009+0000 mon.a (mon.0) 826 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:45:22.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:45:22 smithi088 bash[61791]: ts=2024-09-13T00:45:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:45:23.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:22 smithi077 bash[83127]: cephadm 2024-09-13T00:45:21.571502+0000 mgr.y (mgr.25304) 271 : cephadm [INF] Upgrade: Updating osd.2 2024-09-13T00:45:23.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:22 smithi077 bash[83127]: cephadm 2024-09-13T00:45:21.594258+0000 mgr.y (mgr.25304) 272 : cephadm [INF] Deploying daemon osd.2 on smithi077 2024-09-13T00:45:23.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:22 smithi077 bash[80681]: cephadm 2024-09-13T00:45:21.571502+0000 mgr.y (mgr.25304) 271 : cephadm [INF] Upgrade: Updating osd.2 2024-09-13T00:45:23.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:22 smithi077 bash[80681]: cephadm 2024-09-13T00:45:21.594258+0000 mgr.y (mgr.25304) 272 : cephadm [INF] Deploying daemon osd.2 on smithi077 2024-09-13T00:45:23.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:22 smithi088 bash[68960]: cephadm 2024-09-13T00:45:21.571502+0000 mgr.y (mgr.25304) 271 : cephadm [INF] Upgrade: Updating osd.2 2024-09-13T00:45:23.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:22 smithi088 bash[68960]: cephadm 2024-09-13T00:45:21.594258+0000 mgr.y (mgr.25304) 272 : cephadm [INF] Deploying daemon osd.2 on smithi077 2024-09-13T00:45:24.242 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:23 smithi077 bash[80681]: cluster 2024-09-13T00:45:22.500464+0000 mgr.y (mgr.25304) 273 : cluster [DBG] pgmap v526: 361 pgs: 361 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 47 KiB/s rd, 628 KiB/s wr, 66 op/s 2024-09-13T00:45:24.242 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:23 smithi077 bash[80681]: audit 2024-09-13T00:45:22.993201+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:24.242 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:23 smithi077 bash[80681]: audit 2024-09-13T00:45:23.538380+0000 mon.a (mon.0) 828 : audit [DBG] from='client.? 172.21.15.77:0/3144146768' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:24.242 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:23 smithi077 bash[83127]: cluster 2024-09-13T00:45:22.500464+0000 mgr.y (mgr.25304) 273 : cluster [DBG] pgmap v526: 361 pgs: 361 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 47 KiB/s rd, 628 KiB/s wr, 66 op/s 2024-09-13T00:45:24.243 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:23 smithi077 bash[83127]: audit 2024-09-13T00:45:22.993201+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:24.243 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:23 smithi077 bash[83127]: audit 2024-09-13T00:45:23.538380+0000 mon.a (mon.0) 828 : audit [DBG] from='client.? 172.21.15.77:0/3144146768' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:23 smithi088 bash[68960]: cluster 2024-09-13T00:45:22.500464+0000 mgr.y (mgr.25304) 273 : cluster [DBG] pgmap v526: 361 pgs: 361 active+clean; 4.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 47 KiB/s rd, 628 KiB/s wr, 66 op/s 2024-09-13T00:45:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:23 smithi088 bash[68960]: audit 2024-09-13T00:45:22.993201+0000 mon.a (mon.0) 827 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:23 smithi088 bash[68960]: audit 2024-09-13T00:45:23.538380+0000 mon.a (mon.0) 828 : audit [DBG] from='client.? 172.21.15.77:0/3144146768' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:26.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:25 smithi077 bash[80681]: cluster 2024-09-13T00:45:24.503306+0000 mgr.y (mgr.25304) 274 : cluster [DBG] pgmap v527: 361 pgs: 361 active+clean; 8.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 1.0 MiB/s wr, 27 op/s 2024-09-13T00:45:26.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:25 smithi077 bash[80681]: audit 2024-09-13T00:45:25.817530+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.77:0/1991741314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:26.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:25 smithi077 bash[80681]: cluster 2024-09-13T00:45:25.953430+0000 mon.a (mon.0) 829 : cluster [DBG] osdmap e622: 8 total, 8 up, 8 in 2024-09-13T00:45:26.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:25 smithi077 bash[83127]: cluster 2024-09-13T00:45:24.503306+0000 mgr.y (mgr.25304) 274 : cluster [DBG] pgmap v527: 361 pgs: 361 active+clean; 8.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 1.0 MiB/s wr, 27 op/s 2024-09-13T00:45:26.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:25 smithi077 bash[83127]: audit 2024-09-13T00:45:25.817530+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.77:0/1991741314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:26.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:25 smithi077 bash[83127]: cluster 2024-09-13T00:45:25.953430+0000 mon.a (mon.0) 829 : cluster [DBG] osdmap e622: 8 total, 8 up, 8 in 2024-09-13T00:45:26.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:25 smithi088 bash[68960]: cluster 2024-09-13T00:45:24.503306+0000 mgr.y (mgr.25304) 274 : cluster [DBG] pgmap v527: 361 pgs: 361 active+clean; 8.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 1.0 MiB/s wr, 27 op/s 2024-09-13T00:45:26.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:25 smithi088 bash[68960]: audit 2024-09-13T00:45:25.817530+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.77:0/1991741314' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:26.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:25 smithi088 bash[68960]: cluster 2024-09-13T00:45:25.953430+0000 mon.a (mon.0) 829 : cluster [DBG] osdmap e622: 8 total, 8 up, 8 in 2024-09-13T00:45:26.860 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:26.860 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:45:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:26.861 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:45:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:26.861 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:26.861 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:26.862 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:26.862 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:45:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:26.862 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:45:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:26.863 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:45:26 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:27.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:27 smithi077 bash[83127]: cluster 2024-09-13T00:45:26.963526+0000 mon.a (mon.0) 830 : cluster [INF] osd.2 marked itself down and dead 2024-09-13T00:45:27.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:26 smithi077 bash[80681]: debug 2024-09-13T00:45:26.992+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:45:27.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:27 smithi077 bash[80681]: cluster 2024-09-13T00:45:26.963526+0000 mon.a (mon.0) 830 : cluster [INF] osd.2 marked itself down and dead 2024-09-13T00:45:27.279 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:26 smithi077 systemd[1]: Stopping Ceph osd.2 for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:45:27.279 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:26 smithi077 bash[46201]: debug 2024-09-13T00:45:26.960+0000 7f119cf52640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:45:27.279 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:26 smithi077 bash[46201]: debug 2024-09-13T00:45:26.960+0000 7f119cf52640 -1 osd.2 622 *** Got signal Terminated *** 2024-09-13T00:45:27.279 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:26 smithi077 bash[46201]: debug 2024-09-13T00:45:26.960+0000 7f119cf52640 -1 osd.2 622 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T00:45:27.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:27 smithi088 bash[68960]: cluster 2024-09-13T00:45:26.963526+0000 mon.a (mon.0) 830 : cluster [INF] osd.2 marked itself down and dead 2024-09-13T00:45:28.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:28 smithi077 bash[83127]: cluster 2024-09-13T00:45:26.504472+0000 mgr.y (mgr.25304) 275 : cluster [DBG] pgmap v529: 361 pgs: 361 active+clean; 8.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 1.0 MiB/s wr, 27 op/s 2024-09-13T00:45:28.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:28 smithi077 bash[83127]: cluster 2024-09-13T00:45:26.996258+0000 mon.a (mon.0) 831 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T00:45:28.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:28 smithi077 bash[83127]: cluster 2024-09-13T00:45:26.996278+0000 mon.a (mon.0) 832 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:45:28.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:28 smithi077 bash[83127]: cluster 2024-09-13T00:45:27.009867+0000 mon.a (mon.0) 833 : cluster [DBG] osdmap e623: 8 total, 7 up, 8 in 2024-09-13T00:45:28.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:28 smithi077 bash[80681]: cluster 2024-09-13T00:45:26.504472+0000 mgr.y (mgr.25304) 275 : cluster [DBG] pgmap v529: 361 pgs: 361 active+clean; 8.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 1.0 MiB/s wr, 27 op/s 2024-09-13T00:45:28.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:28 smithi077 bash[80681]: cluster 2024-09-13T00:45:26.996258+0000 mon.a (mon.0) 831 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T00:45:28.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:28 smithi077 bash[80681]: cluster 2024-09-13T00:45:26.996278+0000 mon.a (mon.0) 832 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:45:28.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:28 smithi077 bash[80681]: cluster 2024-09-13T00:45:27.009867+0000 mon.a (mon.0) 833 : cluster [DBG] osdmap e623: 8 total, 7 up, 8 in 2024-09-13T00:45:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:28 smithi088 bash[68960]: cluster 2024-09-13T00:45:26.504472+0000 mgr.y (mgr.25304) 275 : cluster [DBG] pgmap v529: 361 pgs: 361 active+clean; 8.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 1.0 MiB/s wr, 27 op/s 2024-09-13T00:45:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:28 smithi088 bash[68960]: cluster 2024-09-13T00:45:26.996258+0000 mon.a (mon.0) 831 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T00:45:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:28 smithi088 bash[68960]: cluster 2024-09-13T00:45:26.996278+0000 mon.a (mon.0) 832 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:45:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:28 smithi088 bash[68960]: cluster 2024-09-13T00:45:27.009867+0000 mon.a (mon.0) 833 : cluster [DBG] osdmap e623: 8 total, 7 up, 8 in 2024-09-13T00:45:28.666 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:45:28] "GET /metrics HTTP/1.1" 200 55812 "" "Prometheus/2.43.0" 2024-09-13T00:45:28.666 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:28 smithi077 bash[99556]: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-2 2024-09-13T00:45:28.666 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:28 smithi077 bash[99648]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-2 2024-09-13T00:45:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:29 smithi077 bash[80681]: cluster 2024-09-13T00:45:28.011652+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e624: 8 total, 7 up, 8 in 2024-09-13T00:45:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:29 smithi077 bash[80681]: audit 2024-09-13T00:45:28.142854+0000 mon.a (mon.0) 835 : audit [DBG] from='client.? 172.21.15.77:0/484668672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:29 smithi077 bash[80681]: audit 2024-09-13T00:45:28.423969+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:29.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:29 smithi077 bash[80681]: audit 2024-09-13T00:45:28.425783+0000 mon.a (mon.0) 837 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:29.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:29 smithi077 bash[83127]: cluster 2024-09-13T00:45:28.011652+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e624: 8 total, 7 up, 8 in 2024-09-13T00:45:29.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:29 smithi077 bash[83127]: audit 2024-09-13T00:45:28.142854+0000 mon.a (mon.0) 835 : audit [DBG] from='client.? 172.21.15.77:0/484668672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:29.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:29 smithi077 bash[83127]: audit 2024-09-13T00:45:28.423969+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:29.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:29 smithi077 bash[83127]: audit 2024-09-13T00:45:28.425783+0000 mon.a (mon.0) 837 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:29 smithi088 bash[68960]: cluster 2024-09-13T00:45:28.011652+0000 mon.a (mon.0) 834 : cluster [DBG] osdmap e624: 8 total, 7 up, 8 in 2024-09-13T00:45:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:29 smithi088 bash[68960]: audit 2024-09-13T00:45:28.142854+0000 mon.a (mon.0) 835 : audit [DBG] from='client.? 172.21.15.77:0/484668672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:29 smithi088 bash[68960]: audit 2024-09-13T00:45:28.423969+0000 mon.a (mon.0) 836 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:29 smithi088 bash[68960]: audit 2024-09-13T00:45:28.425783+0000 mon.a (mon.0) 837 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:30.706 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:30 smithi077 bash[80681]: cluster 2024-09-13T00:45:28.505224+0000 mgr.y (mgr.25304) 276 : cluster [DBG] pgmap v532: 361 pgs: 35 stale+active+clean, 326 active+clean; 8.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 852 B/s rd, 682 KiB/s wr, 1 op/s 2024-09-13T00:45:30.707 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:30 smithi077 bash[83127]: cluster 2024-09-13T00:45:28.505224+0000 mgr.y (mgr.25304) 276 : cluster [DBG] pgmap v532: 361 pgs: 35 stale+active+clean, 326 active+clean; 8.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 852 B/s rd, 682 KiB/s wr, 1 op/s 2024-09-13T00:45:30.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:30 smithi088 bash[68960]: cluster 2024-09-13T00:45:28.505224+0000 mgr.y (mgr.25304) 276 : cluster [DBG] pgmap v532: 361 pgs: 35 stale+active+clean, 326 active+clean; 8.6 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 852 B/s rd, 682 KiB/s wr, 1 op/s 2024-09-13T00:45:31.527 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:45:31 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:31.528 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:45:31 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:31.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:31 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:31.529 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:45:31 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:31.529 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:45:31 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:31.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:31 smithi077 bash[80681]: audit 2024-09-13T00:45:30.432370+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.77:0/4271576461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:31.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:31 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:31.529 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:31 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-13T00:45:31.530 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:31 smithi077 systemd[1]: Stopped Ceph osd.2 for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:45:31.530 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:31 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:31.530 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:45:31 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:31.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:31 smithi077 bash[83127]: audit 2024-09-13T00:45:30.432370+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.77:0/4271576461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:31.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:31 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:31.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:31 smithi088 bash[68960]: audit 2024-09-13T00:45:30.432370+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.77:0/4271576461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:32.028 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:31 smithi077 systemd[1]: Started Ceph osd.2 for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:45:32.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:45:32 smithi088 bash[61791]: ts=2024-09-13T00:45:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:45:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:32 smithi088 bash[68960]: cluster 2024-09-13T00:45:30.506875+0000 mgr.y (mgr.25304) 277 : cluster [DBG] pgmap v533: 361 pgs: 76 active+undersized, 7 stale+active+clean, 35 active+undersized+degraded, 243 active+clean; 14 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 853 KiB/s wr, 1 op/s; 56/519 objects degraded (10.790%) 2024-09-13T00:45:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:32 smithi088 bash[68960]: cluster 2024-09-13T00:45:31.423608+0000 mon.a (mon.0) 838 : cluster [WRN] Health check failed: Degraded data redundancy: 56/519 objects degraded (10.790%), 35 pgs degraded (PG_DEGRADED) 2024-09-13T00:45:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:32 smithi088 bash[68960]: audit 2024-09-13T00:45:31.606140+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:32 smithi088 bash[68960]: audit 2024-09-13T00:45:31.611686+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:32 smithi088 bash[68960]: audit 2024-09-13T00:45:31.612750+0000 mon.a (mon.0) 841 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:45:32.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:32 smithi077 bash[83127]: cluster 2024-09-13T00:45:30.506875+0000 mgr.y (mgr.25304) 277 : cluster [DBG] pgmap v533: 361 pgs: 76 active+undersized, 7 stale+active+clean, 35 active+undersized+degraded, 243 active+clean; 14 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 853 KiB/s wr, 1 op/s; 56/519 objects degraded (10.790%) 2024-09-13T00:45:32.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:32 smithi077 bash[83127]: cluster 2024-09-13T00:45:31.423608+0000 mon.a (mon.0) 838 : cluster [WRN] Health check failed: Degraded data redundancy: 56/519 objects degraded (10.790%), 35 pgs degraded (PG_DEGRADED) 2024-09-13T00:45:32.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:32 smithi077 bash[83127]: audit 2024-09-13T00:45:31.606140+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:32.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:32 smithi077 bash[83127]: audit 2024-09-13T00:45:31.611686+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:32.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:32 smithi077 bash[83127]: audit 2024-09-13T00:45:31.612750+0000 mon.a (mon.0) 841 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:45:32.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:32 smithi077 bash[80681]: cluster 2024-09-13T00:45:30.506875+0000 mgr.y (mgr.25304) 277 : cluster [DBG] pgmap v533: 361 pgs: 76 active+undersized, 7 stale+active+clean, 35 active+undersized+degraded, 243 active+clean; 14 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 853 KiB/s wr, 1 op/s; 56/519 objects degraded (10.790%) 2024-09-13T00:45:32.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:32 smithi077 bash[80681]: cluster 2024-09-13T00:45:31.423608+0000 mon.a (mon.0) 838 : cluster [WRN] Health check failed: Degraded data redundancy: 56/519 objects degraded (10.790%), 35 pgs degraded (PG_DEGRADED) 2024-09-13T00:45:32.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:32 smithi077 bash[80681]: audit 2024-09-13T00:45:31.606140+0000 mon.a (mon.0) 839 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:32.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:32 smithi077 bash[80681]: audit 2024-09-13T00:45:31.611686+0000 mon.a (mon.0) 840 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:32.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:32 smithi077 bash[80681]: audit 2024-09-13T00:45:31.612750+0000 mon.a (mon.0) 841 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:45:33.622 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:33 smithi077 bash[80681]: audit 2024-09-13T00:45:32.708698+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.77:0/3011817359' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:33.622 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:33 smithi077 bash[83127]: audit 2024-09-13T00:45:32.708698+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.77:0/3011817359' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:33.622 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:33 smithi077 bash[99822]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T00:45:33.623 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:33 smithi077 bash[99822]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-13T00:45:33.623 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:33 smithi077 bash[99822]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-13T00:45:33.623 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:33 smithi077 bash[99822]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-13T00:45:33.623 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:33 smithi077 bash[99822]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-13T00:45:33.623 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:33 smithi077 bash[99822]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-13T00:45:33.623 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:33 smithi077 bash[99822]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-13T00:45:33.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:33 smithi088 bash[68960]: audit 2024-09-13T00:45:32.708698+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.77:0/3011817359' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:34.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:34 smithi077 bash[80681]: cluster 2024-09-13T00:45:32.507851+0000 mgr.y (mgr.25304) 278 : cluster [DBG] pgmap v534: 361 pgs: 90 active+undersized, 41 active+undersized+degraded, 230 active+clean; 14 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 781 KiB/s wr, 1 op/s; 68/519 objects degraded (13.102%) 2024-09-13T00:45:34.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:34 smithi077 bash[83127]: cluster 2024-09-13T00:45:32.507851+0000 mgr.y (mgr.25304) 278 : cluster [DBG] pgmap v534: 361 pgs: 90 active+undersized, 41 active+undersized+degraded, 230 active+clean; 14 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 781 KiB/s wr, 1 op/s; 68/519 objects degraded (13.102%) 2024-09-13T00:45:34.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:34 smithi088 bash[68960]: cluster 2024-09-13T00:45:32.507851+0000 mgr.y (mgr.25304) 278 : cluster [DBG] pgmap v534: 361 pgs: 90 active+undersized, 41 active+undersized+degraded, 230 active+clean; 14 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 781 KiB/s wr, 1 op/s; 68/519 objects degraded (13.102%) 2024-09-13T00:45:35.778 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:35 smithi077 bash[100067]: debug 2024-09-13T00:45:35.608+0000 7fadbaf40640 1 -- 172.21.15.77:0/4291578940 <== mon.0 v2:172.21.15.77:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5610d95851e0 con 0x5610d9545c00 2024-09-13T00:45:35.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:35 smithi077 bash[80681]: cluster 2024-09-13T00:45:34.511155+0000 mgr.y (mgr.25304) 279 : cluster [DBG] pgmap v535: 361 pgs: 90 active+undersized, 41 active+undersized+degraded, 230 active+clean; 18 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 1 op/s; 68/522 objects degraded (13.027%) 2024-09-13T00:45:35.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:35 smithi077 bash[80681]: audit 2024-09-13T00:45:34.985871+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.77:0/3486777928' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:35.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:35 smithi077 bash[83127]: cluster 2024-09-13T00:45:34.511155+0000 mgr.y (mgr.25304) 279 : cluster [DBG] pgmap v535: 361 pgs: 90 active+undersized, 41 active+undersized+degraded, 230 active+clean; 18 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 1 op/s; 68/522 objects degraded (13.027%) 2024-09-13T00:45:35.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:35 smithi077 bash[83127]: audit 2024-09-13T00:45:34.985871+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.77:0/3486777928' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:35.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:35 smithi088 bash[68960]: cluster 2024-09-13T00:45:34.511155+0000 mgr.y (mgr.25304) 279 : cluster [DBG] pgmap v535: 361 pgs: 90 active+undersized, 41 active+undersized+degraded, 230 active+clean; 18 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 1 op/s; 68/522 objects degraded (13.027%) 2024-09-13T00:45:35.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:35 smithi088 bash[68960]: audit 2024-09-13T00:45:34.985871+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.77:0/3486777928' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:36.456 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:36 smithi077 bash[100067]: debug 2024-09-13T00:45:36.152+0000 7fadbd7a6640 -1 Falling back to public interface 2024-09-13T00:45:36.476 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.RemoveFullTry (23997 ms) 2024-09-13T00:45:36.476 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.RemoveFullTryDataPool 2024-09-13T00:45:36.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:36 smithi077 bash[80681]: cluster 2024-09-13T00:45:35.948878+0000 mon.a (mon.0) 842 : cluster [WRN] pool 'test-librbd-smithi077-68798-22' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T00:45:36.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:36 smithi077 bash[80681]: cluster 2024-09-13T00:45:35.949517+0000 mon.a (mon.0) 843 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T00:45:36.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:36 smithi077 bash[80681]: cluster 2024-09-13T00:45:35.955129+0000 mon.a (mon.0) 844 : cluster [DBG] osdmap e625: 8 total, 7 up, 8 in 2024-09-13T00:45:36.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:36 smithi077 bash[83127]: cluster 2024-09-13T00:45:35.948878+0000 mon.a (mon.0) 842 : cluster [WRN] pool 'test-librbd-smithi077-68798-22' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T00:45:36.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:36 smithi077 bash[83127]: cluster 2024-09-13T00:45:35.949517+0000 mon.a (mon.0) 843 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T00:45:36.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:36 smithi077 bash[83127]: cluster 2024-09-13T00:45:35.955129+0000 mon.a (mon.0) 844 : cluster [DBG] osdmap e625: 8 total, 7 up, 8 in 2024-09-13T00:45:36.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:36 smithi088 bash[68960]: cluster 2024-09-13T00:45:35.948878+0000 mon.a (mon.0) 842 : cluster [WRN] pool 'test-librbd-smithi077-68798-22' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T00:45:36.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:36 smithi088 bash[68960]: cluster 2024-09-13T00:45:35.949517+0000 mon.a (mon.0) 843 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T00:45:36.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:36 smithi088 bash[68960]: cluster 2024-09-13T00:45:35.955129+0000 mon.a (mon.0) 844 : cluster [DBG] osdmap e625: 8 total, 7 up, 8 in 2024-09-13T00:45:37.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:37 smithi077 bash[83127]: cluster 2024-09-13T00:45:36.512074+0000 mgr.y (mgr.25304) 280 : cluster [DBG] pgmap v537: 361 pgs: 90 active+undersized, 41 active+undersized+degraded, 230 active+clean; 18 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 1 op/s; 68/522 objects degraded (13.027%) 2024-09-13T00:45:37.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:37 smithi077 bash[83127]: audit 2024-09-13T00:45:37.273102+0000 mon.a (mon.0) 845 : audit [DBG] from='client.? 172.21.15.77:0/3871629816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:37.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:37 smithi077 bash[83127]: audit 2024-09-13T00:45:37.436909+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:37.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:37 smithi077 bash[83127]: audit 2024-09-13T00:45:37.447026+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:37.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:37 smithi077 bash[80681]: cluster 2024-09-13T00:45:36.512074+0000 mgr.y (mgr.25304) 280 : cluster [DBG] pgmap v537: 361 pgs: 90 active+undersized, 41 active+undersized+degraded, 230 active+clean; 18 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 1 op/s; 68/522 objects degraded (13.027%) 2024-09-13T00:45:37.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:37 smithi077 bash[80681]: audit 2024-09-13T00:45:37.273102+0000 mon.a (mon.0) 845 : audit [DBG] from='client.? 172.21.15.77:0/3871629816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:37.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:37 smithi077 bash[80681]: audit 2024-09-13T00:45:37.436909+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:37.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:37 smithi077 bash[80681]: audit 2024-09-13T00:45:37.447026+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:37.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:37 smithi088 bash[68960]: cluster 2024-09-13T00:45:36.512074+0000 mgr.y (mgr.25304) 280 : cluster [DBG] pgmap v537: 361 pgs: 90 active+undersized, 41 active+undersized+degraded, 230 active+clean; 18 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s wr, 1 op/s; 68/522 objects degraded (13.027%) 2024-09-13T00:45:37.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:37 smithi088 bash[68960]: audit 2024-09-13T00:45:37.273102+0000 mon.a (mon.0) 845 : audit [DBG] from='client.? 172.21.15.77:0/3871629816' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:37.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:37 smithi088 bash[68960]: audit 2024-09-13T00:45:37.436909+0000 mon.a (mon.0) 846 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:37.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:37 smithi088 bash[68960]: audit 2024-09-13T00:45:37.447026+0000 mon.a (mon.0) 847 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:38.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:45:38] "GET /metrics HTTP/1.1" 200 55812 "" "Prometheus/2.43.0" 2024-09-13T00:45:38.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:38 smithi077 bash[83127]: cluster 2024-09-13T00:45:37.477715+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e626: 8 total, 7 up, 8 in 2024-09-13T00:45:38.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:38 smithi077 bash[83127]: audit 2024-09-13T00:45:37.479102+0000 mon.c (mon.2) 35 : audit [INF] from='client.? 172.21.15.77:0/2398008294' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:38.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:38 smithi077 bash[83127]: audit 2024-09-13T00:45:37.483036+0000 mon.a (mon.0) 849 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:38.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:38 smithi077 bash[83127]: audit 2024-09-13T00:45:38.345508+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:38.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:38 smithi077 bash[83127]: audit 2024-09-13T00:45:38.350680+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:38.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:38 smithi077 bash[80681]: cluster 2024-09-13T00:45:37.477715+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e626: 8 total, 7 up, 8 in 2024-09-13T00:45:38.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:38 smithi077 bash[80681]: audit 2024-09-13T00:45:37.479102+0000 mon.c (mon.2) 35 : audit [INF] from='client.? 172.21.15.77:0/2398008294' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:38.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:38 smithi077 bash[80681]: audit 2024-09-13T00:45:37.483036+0000 mon.a (mon.0) 849 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:38.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:38 smithi077 bash[80681]: audit 2024-09-13T00:45:38.345508+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:38.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:38 smithi077 bash[80681]: audit 2024-09-13T00:45:38.350680+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:38.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:38 smithi088 bash[68960]: cluster 2024-09-13T00:45:37.477715+0000 mon.a (mon.0) 848 : cluster [DBG] osdmap e626: 8 total, 7 up, 8 in 2024-09-13T00:45:38.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:38 smithi088 bash[68960]: audit 2024-09-13T00:45:37.479102+0000 mon.c (mon.2) 35 : audit [INF] from='client.? 172.21.15.77:0/2398008294' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:38.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:38 smithi088 bash[68960]: audit 2024-09-13T00:45:37.483036+0000 mon.a (mon.0) 849 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:38.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:38 smithi088 bash[68960]: audit 2024-09-13T00:45:38.345508+0000 mon.a (mon.0) 850 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:38.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:38 smithi088 bash[68960]: audit 2024-09-13T00:45:38.350680+0000 mon.a (mon.0) 851 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:39.778 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:39 smithi077 bash[100067]: debug 2024-09-13T00:45:39.404+0000 7fadbd7a6640 -1 osd.2 622 log_to_monitors true 2024-09-13T00:45:39.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:39 smithi077 bash[80681]: audit 2024-09-13T00:45:38.479560+0000 mon.a (mon.0) 852 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:45:39.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:39 smithi077 bash[80681]: cluster 2024-09-13T00:45:38.482092+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e627: 8 total, 7 up, 8 in 2024-09-13T00:45:39.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:39 smithi077 bash[80681]: cluster 2024-09-13T00:45:38.514259+0000 mgr.y (mgr.25304) 281 : cluster [DBG] pgmap v540: 393 pgs: 32 unknown, 90 active+undersized, 41 active+undersized+degraded, 230 active+clean; 18 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 852 B/s rd, 682 KiB/s wr, 1 op/s; 68/522 objects degraded (13.027%) 2024-09-13T00:45:39.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:39 smithi077 bash[80681]: audit 2024-09-13T00:45:39.067566+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:39.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:39 smithi077 bash[80681]: audit 2024-09-13T00:45:39.072449+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:39.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:39 smithi077 bash[80681]: audit 2024-09-13T00:45:39.410045+0000 mon.a (mon.0) 856 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/3472170327,v1:172.21.15.77:6819/3472170327]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T00:45:39.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:39 smithi077 bash[83127]: audit 2024-09-13T00:45:38.479560+0000 mon.a (mon.0) 852 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:45:39.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:39 smithi077 bash[83127]: cluster 2024-09-13T00:45:38.482092+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e627: 8 total, 7 up, 8 in 2024-09-13T00:45:39.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:39 smithi077 bash[83127]: cluster 2024-09-13T00:45:38.514259+0000 mgr.y (mgr.25304) 281 : cluster [DBG] pgmap v540: 393 pgs: 32 unknown, 90 active+undersized, 41 active+undersized+degraded, 230 active+clean; 18 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 852 B/s rd, 682 KiB/s wr, 1 op/s; 68/522 objects degraded (13.027%) 2024-09-13T00:45:39.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:39 smithi077 bash[83127]: audit 2024-09-13T00:45:39.067566+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:39.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:39 smithi077 bash[83127]: audit 2024-09-13T00:45:39.072449+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:39.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:39 smithi077 bash[83127]: audit 2024-09-13T00:45:39.410045+0000 mon.a (mon.0) 856 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/3472170327,v1:172.21.15.77:6819/3472170327]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T00:45:39.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:39 smithi088 bash[68960]: audit 2024-09-13T00:45:38.479560+0000 mon.a (mon.0) 852 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:45:39.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:39 smithi088 bash[68960]: cluster 2024-09-13T00:45:38.482092+0000 mon.a (mon.0) 853 : cluster [DBG] osdmap e627: 8 total, 7 up, 8 in 2024-09-13T00:45:39.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:39 smithi088 bash[68960]: cluster 2024-09-13T00:45:38.514259+0000 mgr.y (mgr.25304) 281 : cluster [DBG] pgmap v540: 393 pgs: 32 unknown, 90 active+undersized, 41 active+undersized+degraded, 230 active+clean; 18 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 852 B/s rd, 682 KiB/s wr, 1 op/s; 68/522 objects degraded (13.027%) 2024-09-13T00:45:39.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:39 smithi088 bash[68960]: audit 2024-09-13T00:45:39.067566+0000 mon.a (mon.0) 854 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:39.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:39 smithi088 bash[68960]: audit 2024-09-13T00:45:39.072449+0000 mon.a (mon.0) 855 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:39.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:39 smithi088 bash[68960]: audit 2024-09-13T00:45:39.410045+0000 mon.a (mon.0) 856 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/3472170327,v1:172.21.15.77:6819/3472170327]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-13T00:45:40.502 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:40 smithi077 bash[100067]: debug 2024-09-13T00:45:40.048+0000 7fadb1548640 -1 osd.2 622 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T00:45:40.546 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:40 smithi077 bash[80681]: debug 2024-09-13T00:45:40.500+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:45:40.546 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:40 smithi077 bash[80681]: audit 2024-09-13T00:45:39.507712+0000 mon.a (mon.0) 857 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/3472170327,v1:172.21.15.77:6819/3472170327]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T00:45:40.546 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:40 smithi077 bash[80681]: cluster 2024-09-13T00:45:39.507857+0000 mon.a (mon.0) 858 : cluster [DBG] osdmap e628: 8 total, 7 up, 8 in 2024-09-13T00:45:40.546 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:40 smithi077 bash[80681]: audit 2024-09-13T00:45:39.516081+0000 mon.c (mon.2) 36 : audit [INF] from='client.? 172.21.15.77:0/3592010428' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:40.546 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:40 smithi077 bash[80681]: audit 2024-09-13T00:45:39.517274+0000 mon.a (mon.0) 859 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:40.546 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:40 smithi077 bash[80681]: audit 2024-09-13T00:45:39.524664+0000 mon.a (mon.0) 860 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/3472170327,v1:172.21.15.77:6819/3472170327]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:45:40.547 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:40 smithi077 bash[83127]: audit 2024-09-13T00:45:39.507712+0000 mon.a (mon.0) 857 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/3472170327,v1:172.21.15.77:6819/3472170327]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T00:45:40.547 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:40 smithi077 bash[83127]: cluster 2024-09-13T00:45:39.507857+0000 mon.a (mon.0) 858 : cluster [DBG] osdmap e628: 8 total, 7 up, 8 in 2024-09-13T00:45:40.547 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:40 smithi077 bash[83127]: audit 2024-09-13T00:45:39.516081+0000 mon.c (mon.2) 36 : audit [INF] from='client.? 172.21.15.77:0/3592010428' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:40.547 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:40 smithi077 bash[83127]: audit 2024-09-13T00:45:39.517274+0000 mon.a (mon.0) 859 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:40.547 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:40 smithi077 bash[83127]: audit 2024-09-13T00:45:39.524664+0000 mon.a (mon.0) 860 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/3472170327,v1:172.21.15.77:6819/3472170327]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:45:40.547 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:40 smithi077 bash[83127]: audit 2024-09-13T00:45:39.597364+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.77:0/1338482087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:40.547 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:40 smithi077 bash[83127]: audit 2024-09-13T00:45:40.035646+0000 mon.a (mon.0) 861 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/3472170327,v1:172.21.15.77:6819/3472170327]' entity='osd.2' 2024-09-13T00:45:40.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:40 smithi077 bash[80681]: audit 2024-09-13T00:45:39.597364+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.77:0/1338482087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:40.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:40 smithi077 bash[80681]: audit 2024-09-13T00:45:40.035646+0000 mon.a (mon.0) 861 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/3472170327,v1:172.21.15.77:6819/3472170327]' entity='osd.2' 2024-09-13T00:45:40.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:40 smithi088 bash[68960]: audit 2024-09-13T00:45:39.507712+0000 mon.a (mon.0) 857 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/3472170327,v1:172.21.15.77:6819/3472170327]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-13T00:45:40.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:40 smithi088 bash[68960]: cluster 2024-09-13T00:45:39.507857+0000 mon.a (mon.0) 858 : cluster [DBG] osdmap e628: 8 total, 7 up, 8 in 2024-09-13T00:45:40.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:40 smithi088 bash[68960]: audit 2024-09-13T00:45:39.516081+0000 mon.c (mon.2) 36 : audit [INF] from='client.? 172.21.15.77:0/3592010428' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:40.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:40 smithi088 bash[68960]: audit 2024-09-13T00:45:39.517274+0000 mon.a (mon.0) 859 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:45:40.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:40 smithi088 bash[68960]: audit 2024-09-13T00:45:39.524664+0000 mon.a (mon.0) 860 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/3472170327,v1:172.21.15.77:6819/3472170327]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:45:40.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:40 smithi088 bash[68960]: audit 2024-09-13T00:45:39.597364+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.77:0/1338482087' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:40.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:40 smithi088 bash[68960]: audit 2024-09-13T00:45:40.035646+0000 mon.a (mon.0) 861 : audit [INF] from='osd.2 [v2:172.21.15.77:6818/3472170327,v1:172.21.15.77:6819/3472170327]' entity='osd.2' 2024-09-13T00:45:41.646 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:45:41.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:41 smithi088 bash[68960]: cluster 2024-09-13T00:45:40.505868+0000 mon.a (mon.0) 862 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:45:41.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:41 smithi088 bash[68960]: cluster 2024-09-13T00:45:40.505926+0000 mon.a (mon.0) 863 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T00:45:41.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:41 smithi088 bash[68960]: cluster 2024-09-13T00:45:40.516194+0000 mgr.y (mgr.25304) 282 : cluster [DBG] pgmap v543: 425 pgs: 38 unknown, 102 active+undersized, 40 active+undersized+degraded, 245 active+clean; 4.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 513 KiB/s wr, 39 op/s; 67/501 objects degraded (13.373%) 2024-09-13T00:45:41.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:41 smithi088 bash[68960]: audit 2024-09-13T00:45:40.531145+0000 mon.a (mon.0) 864 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:45:41.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:41 smithi088 bash[68960]: cluster 2024-09-13T00:45:40.533587+0000 mon.a (mon.0) 865 : cluster [INF] osd.2 [v2:172.21.15.77:6818/3472170327,v1:172.21.15.77:6819/3472170327] boot 2024-09-13T00:45:41.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:41 smithi088 bash[68960]: cluster 2024-09-13T00:45:40.533679+0000 mon.a (mon.0) 866 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2024-09-13T00:45:41.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:41 smithi088 bash[68960]: audit 2024-09-13T00:45:40.538352+0000 mon.a (mon.0) 867 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:45:41.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:41 smithi088 bash[68960]: cluster 2024-09-13T00:45:40.950948+0000 mon.a (mon.0) 868 : cluster [WRN] Health check update: Degraded data redundancy: 68/522 objects degraded (13.027%), 41 pgs degraded (PG_DEGRADED) 2024-09-13T00:45:41.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:41 smithi088 bash[68960]: cluster 2024-09-13T00:45:41.530661+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2024-09-13T00:45:42.028 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[73137]: [13/Sep/2024:00:45:41] ENGINE Bus STOPPING 2024-09-13T00:45:42.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[83127]: cluster 2024-09-13T00:45:40.505868+0000 mon.a (mon.0) 862 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:45:42.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[83127]: cluster 2024-09-13T00:45:40.505926+0000 mon.a (mon.0) 863 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T00:45:42.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[83127]: cluster 2024-09-13T00:45:40.516194+0000 mgr.y (mgr.25304) 282 : cluster [DBG] pgmap v543: 425 pgs: 38 unknown, 102 active+undersized, 40 active+undersized+degraded, 245 active+clean; 4.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 513 KiB/s wr, 39 op/s; 67/501 objects degraded (13.373%) 2024-09-13T00:45:42.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[83127]: audit 2024-09-13T00:45:40.531145+0000 mon.a (mon.0) 864 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:45:42.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[83127]: cluster 2024-09-13T00:45:40.533587+0000 mon.a (mon.0) 865 : cluster [INF] osd.2 [v2:172.21.15.77:6818/3472170327,v1:172.21.15.77:6819/3472170327] boot 2024-09-13T00:45:42.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[83127]: cluster 2024-09-13T00:45:40.533679+0000 mon.a (mon.0) 866 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2024-09-13T00:45:42.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[83127]: audit 2024-09-13T00:45:40.538352+0000 mon.a (mon.0) 867 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:45:42.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[83127]: cluster 2024-09-13T00:45:40.950948+0000 mon.a (mon.0) 868 : cluster [WRN] Health check update: Degraded data redundancy: 68/522 objects degraded (13.027%), 41 pgs degraded (PG_DEGRADED) 2024-09-13T00:45:42.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[83127]: cluster 2024-09-13T00:45:41.530661+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2024-09-13T00:45:42.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[80681]: cluster 2024-09-13T00:45:40.505868+0000 mon.a (mon.0) 862 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:45:42.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[80681]: cluster 2024-09-13T00:45:40.505926+0000 mon.a (mon.0) 863 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T00:45:42.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[80681]: cluster 2024-09-13T00:45:40.516194+0000 mgr.y (mgr.25304) 282 : cluster [DBG] pgmap v543: 425 pgs: 38 unknown, 102 active+undersized, 40 active+undersized+degraded, 245 active+clean; 4.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 513 KiB/s wr, 39 op/s; 67/501 objects degraded (13.373%) 2024-09-13T00:45:42.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[80681]: audit 2024-09-13T00:45:40.531145+0000 mon.a (mon.0) 864 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:45:42.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[80681]: cluster 2024-09-13T00:45:40.533587+0000 mon.a (mon.0) 865 : cluster [INF] osd.2 [v2:172.21.15.77:6818/3472170327,v1:172.21.15.77:6819/3472170327] boot 2024-09-13T00:45:42.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[80681]: cluster 2024-09-13T00:45:40.533679+0000 mon.a (mon.0) 866 : cluster [DBG] osdmap e629: 8 total, 8 up, 8 in 2024-09-13T00:45:42.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[80681]: audit 2024-09-13T00:45:40.538352+0000 mon.a (mon.0) 867 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-13T00:45:42.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[80681]: cluster 2024-09-13T00:45:40.950948+0000 mon.a (mon.0) 868 : cluster [WRN] Health check update: Degraded data redundancy: 68/522 objects degraded (13.027%), 41 pgs degraded (PG_DEGRADED) 2024-09-13T00:45:42.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:41 smithi077 bash[80681]: cluster 2024-09-13T00:45:41.530661+0000 mon.a (mon.0) 869 : cluster [DBG] osdmap e630: 8 total, 8 up, 8 in 2024-09-13T00:45:42.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:42 smithi077 bash[73137]: [13/Sep/2024:00:45:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:45:42.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:42 smithi077 bash[73137]: [13/Sep/2024:00:45:42] ENGINE Bus STOPPED 2024-09-13T00:45:42.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:42 smithi077 bash[73137]: [13/Sep/2024:00:45:42] ENGINE Bus STARTING 2024-09-13T00:45:42.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:42 smithi077 bash[73137]: [13/Sep/2024:00:45:42] ENGINE Serving on http://:::9283 2024-09-13T00:45:42.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:42 smithi077 bash[73137]: [13/Sep/2024:00:45:42] ENGINE Bus STARTED 2024-09-13T00:45:42.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:45:42 smithi088 bash[61791]: ts=2024-09-13T00:45:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:45:43.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:42 smithi077 bash[80681]: audit 2024-09-13T00:45:41.649390+0000 mon.a (mon.0) 870 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' 2024-09-13T00:45:43.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:42 smithi077 bash[80681]: audit 2024-09-13T00:45:41.894099+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.77:0/1815657033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:43.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:42 smithi077 bash[80681]: cluster 2024-09-13T00:45:42.557370+0000 mon.a (mon.0) 872 : cluster [DBG] osdmap e631: 8 total, 8 up, 8 in 2024-09-13T00:45:43.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:42 smithi077 bash[83127]: audit 2024-09-13T00:45:41.649390+0000 mon.a (mon.0) 870 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' 2024-09-13T00:45:43.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:42 smithi077 bash[83127]: audit 2024-09-13T00:45:41.894099+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.77:0/1815657033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:43.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:42 smithi077 bash[83127]: cluster 2024-09-13T00:45:42.557370+0000 mon.a (mon.0) 872 : cluster [DBG] osdmap e631: 8 total, 8 up, 8 in 2024-09-13T00:45:43.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:42 smithi088 bash[68960]: audit 2024-09-13T00:45:41.649390+0000 mon.a (mon.0) 870 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' 2024-09-13T00:45:43.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:42 smithi088 bash[68960]: audit 2024-09-13T00:45:41.894099+0000 mon.a (mon.0) 871 : audit [DBG] from='client.? 172.21.15.77:0/1815657033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:43.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:42 smithi088 bash[68960]: cluster 2024-09-13T00:45:42.557370+0000 mon.a (mon.0) 872 : cluster [DBG] osdmap e631: 8 total, 8 up, 8 in 2024-09-13T00:45:44.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:44 smithi077 bash[80681]: cluster 2024-09-13T00:45:42.517647+0000 mgr.y (mgr.25304) 283 : cluster [DBG] pgmap v545: 425 pgs: 2 creating+peering, 16 peering, 27 unknown, 93 active+undersized, 36 active+undersized+degraded, 251 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 508 KiB/s wr, 39 op/s; 57/498 objects degraded (11.446%) 2024-09-13T00:45:44.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:44 smithi077 bash[80681]: audit 2024-09-13T00:45:43.424001+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:44.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:44 smithi077 bash[80681]: audit 2024-09-13T00:45:43.436575+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:44.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:44 smithi077 bash[80681]: cluster 2024-09-13T00:45:43.559960+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e632: 8 total, 8 up, 8 in 2024-09-13T00:45:44.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:44 smithi077 bash[80681]: audit 2024-09-13T00:45:43.605554+0000 mon.a (mon.0) 876 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi077-68798-24", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:45:44.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:44 smithi077 bash[80681]: audit 2024-09-13T00:45:44.190681+0000 mon.a (mon.0) 877 : audit [DBG] from='client.? 172.21.15.77:0/131933672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:44.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:44 smithi077 bash[83127]: cluster 2024-09-13T00:45:42.517647+0000 mgr.y (mgr.25304) 283 : cluster [DBG] pgmap v545: 425 pgs: 2 creating+peering, 16 peering, 27 unknown, 93 active+undersized, 36 active+undersized+degraded, 251 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 508 KiB/s wr, 39 op/s; 57/498 objects degraded (11.446%) 2024-09-13T00:45:44.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:44 smithi077 bash[83127]: audit 2024-09-13T00:45:43.424001+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:44.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:44 smithi077 bash[83127]: audit 2024-09-13T00:45:43.436575+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:44.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:44 smithi077 bash[83127]: cluster 2024-09-13T00:45:43.559960+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e632: 8 total, 8 up, 8 in 2024-09-13T00:45:44.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:44 smithi077 bash[83127]: audit 2024-09-13T00:45:43.605554+0000 mon.a (mon.0) 876 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi077-68798-24", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:45:44.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:44 smithi077 bash[83127]: audit 2024-09-13T00:45:44.190681+0000 mon.a (mon.0) 877 : audit [DBG] from='client.? 172.21.15.77:0/131933672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:44 smithi088 bash[68960]: cluster 2024-09-13T00:45:42.517647+0000 mgr.y (mgr.25304) 283 : cluster [DBG] pgmap v545: 425 pgs: 2 creating+peering, 16 peering, 27 unknown, 93 active+undersized, 36 active+undersized+degraded, 251 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 508 KiB/s wr, 39 op/s; 57/498 objects degraded (11.446%) 2024-09-13T00:45:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:44 smithi088 bash[68960]: audit 2024-09-13T00:45:43.424001+0000 mon.a (mon.0) 873 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:44 smithi088 bash[68960]: audit 2024-09-13T00:45:43.436575+0000 mon.a (mon.0) 874 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:44 smithi088 bash[68960]: cluster 2024-09-13T00:45:43.559960+0000 mon.a (mon.0) 875 : cluster [DBG] osdmap e632: 8 total, 8 up, 8 in 2024-09-13T00:45:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:44 smithi088 bash[68960]: audit 2024-09-13T00:45:43.605554+0000 mon.a (mon.0) 876 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi077-68798-24", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-09-13T00:45:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:44 smithi088 bash[68960]: audit 2024-09-13T00:45:44.190681+0000 mon.a (mon.0) 877 : audit [DBG] from='client.? 172.21.15.77:0/131933672' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:45.822 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:45 smithi077 bash[80681]: cluster 2024-09-13T00:45:44.520901+0000 mgr.y (mgr.25304) 284 : cluster [DBG] pgmap v548: 425 pgs: 2 creating+peering, 16 peering, 2 unknown, 31 active+undersized, 9 active+undersized+degraded, 365 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 1.2 KiB/s wr, 2 op/s; 14/510 objects degraded (2.745%); 0 B/s, 0 objects/s recovering 2024-09-13T00:45:45.823 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:45 smithi077 bash[80681]: audit 2024-09-13T00:45:44.565430+0000 mon.a (mon.0) 878 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi077-68798-24", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T00:45:45.823 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:45 smithi077 bash[80681]: cluster 2024-09-13T00:45:44.565696+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e633: 8 total, 8 up, 8 in 2024-09-13T00:45:45.823 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:45 smithi077 bash[83127]: cluster 2024-09-13T00:45:44.520901+0000 mgr.y (mgr.25304) 284 : cluster [DBG] pgmap v548: 425 pgs: 2 creating+peering, 16 peering, 2 unknown, 31 active+undersized, 9 active+undersized+degraded, 365 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 1.2 KiB/s wr, 2 op/s; 14/510 objects degraded (2.745%); 0 B/s, 0 objects/s recovering 2024-09-13T00:45:45.823 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:45 smithi077 bash[83127]: audit 2024-09-13T00:45:44.565430+0000 mon.a (mon.0) 878 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi077-68798-24", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T00:45:45.823 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:45 smithi077 bash[83127]: cluster 2024-09-13T00:45:44.565696+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e633: 8 total, 8 up, 8 in 2024-09-13T00:45:45.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:45 smithi088 bash[68960]: cluster 2024-09-13T00:45:44.520901+0000 mgr.y (mgr.25304) 284 : cluster [DBG] pgmap v548: 425 pgs: 2 creating+peering, 16 peering, 2 unknown, 31 active+undersized, 9 active+undersized+degraded, 365 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 1.2 KiB/s wr, 2 op/s; 14/510 objects degraded (2.745%); 0 B/s, 0 objects/s recovering 2024-09-13T00:45:45.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:45 smithi088 bash[68960]: audit 2024-09-13T00:45:44.565430+0000 mon.a (mon.0) 878 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi077-68798-24", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-13T00:45:45.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:45 smithi088 bash[68960]: cluster 2024-09-13T00:45:44.565696+0000 mon.a (mon.0) 879 : cluster [DBG] osdmap e633: 8 total, 8 up, 8 in 2024-09-13T00:45:46.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:46 smithi088 bash[68960]: cluster 2024-09-13T00:45:45.952095+0000 mon.a (mon.0) 880 : cluster [WRN] Health check update: Degraded data redundancy: 14/510 objects degraded (2.745%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T00:45:46.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:46 smithi088 bash[68960]: cluster 2024-09-13T00:45:45.952386+0000 mon.a (mon.0) 881 : cluster [INF] pool 'test-librbd-smithi077-68798-22' no longer out of quota; removing NO_QUOTA flag 2024-09-13T00:45:46.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:46 smithi088 bash[68960]: cluster 2024-09-13T00:45:45.952756+0000 mon.a (mon.0) 882 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T00:45:46.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:46 smithi088 bash[68960]: cluster 2024-09-13T00:45:45.958745+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e634: 8 total, 8 up, 8 in 2024-09-13T00:45:46.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:46 smithi088 bash[68960]: audit 2024-09-13T00:45:46.480866+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.77:0/3445024949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:46.882 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:46 smithi077 bash[83127]: cluster 2024-09-13T00:45:45.952095+0000 mon.a (mon.0) 880 : cluster [WRN] Health check update: Degraded data redundancy: 14/510 objects degraded (2.745%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T00:45:46.882 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:46 smithi077 bash[83127]: cluster 2024-09-13T00:45:45.952386+0000 mon.a (mon.0) 881 : cluster [INF] pool 'test-librbd-smithi077-68798-22' no longer out of quota; removing NO_QUOTA flag 2024-09-13T00:45:46.882 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:46 smithi077 bash[83127]: cluster 2024-09-13T00:45:45.952756+0000 mon.a (mon.0) 882 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T00:45:46.882 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:46 smithi077 bash[83127]: cluster 2024-09-13T00:45:45.958745+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e634: 8 total, 8 up, 8 in 2024-09-13T00:45:46.882 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:46 smithi077 bash[83127]: audit 2024-09-13T00:45:46.480866+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.77:0/3445024949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:46.883 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:46 smithi077 bash[80681]: cluster 2024-09-13T00:45:45.952095+0000 mon.a (mon.0) 880 : cluster [WRN] Health check update: Degraded data redundancy: 14/510 objects degraded (2.745%), 9 pgs degraded (PG_DEGRADED) 2024-09-13T00:45:46.883 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:46 smithi077 bash[80681]: cluster 2024-09-13T00:45:45.952386+0000 mon.a (mon.0) 881 : cluster [INF] pool 'test-librbd-smithi077-68798-22' no longer out of quota; removing NO_QUOTA flag 2024-09-13T00:45:46.883 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:46 smithi077 bash[80681]: cluster 2024-09-13T00:45:45.952756+0000 mon.a (mon.0) 882 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T00:45:46.883 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:46 smithi077 bash[80681]: cluster 2024-09-13T00:45:45.958745+0000 mon.a (mon.0) 883 : cluster [DBG] osdmap e634: 8 total, 8 up, 8 in 2024-09-13T00:45:46.883 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:46 smithi077 bash[80681]: audit 2024-09-13T00:45:46.480866+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.77:0/3445024949' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:47.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:47 smithi088 bash[68960]: cluster 2024-09-13T00:45:46.521891+0000 mgr.y (mgr.25304) 285 : cluster [DBG] pgmap v551: 425 pgs: 425 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.7 KiB/s wr, 3 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:45:47.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:47 smithi088 bash[68960]: cluster 2024-09-13T00:45:46.954739+0000 mon.a (mon.0) 884 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/510 objects degraded (2.745%), 9 pgs degraded) 2024-09-13T00:45:48.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:47 smithi077 bash[83127]: cluster 2024-09-13T00:45:46.521891+0000 mgr.y (mgr.25304) 285 : cluster [DBG] pgmap v551: 425 pgs: 425 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.7 KiB/s wr, 3 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:45:48.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:47 smithi077 bash[83127]: cluster 2024-09-13T00:45:46.954739+0000 mon.a (mon.0) 884 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/510 objects degraded (2.745%), 9 pgs degraded) 2024-09-13T00:45:48.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:47 smithi077 bash[80681]: cluster 2024-09-13T00:45:46.521891+0000 mgr.y (mgr.25304) 285 : cluster [DBG] pgmap v551: 425 pgs: 425 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.7 KiB/s wr, 3 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:45:48.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:47 smithi077 bash[80681]: cluster 2024-09-13T00:45:46.954739+0000 mon.a (mon.0) 884 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/510 objects degraded (2.745%), 9 pgs degraded) 2024-09-13T00:45:48.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:45:48] "GET /metrics HTTP/1.1" 200 60459 "" "Prometheus/2.43.0" 2024-09-13T00:45:49.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:49 smithi077 bash[80681]: cluster 2024-09-13T00:45:48.522473+0000 mgr.y (mgr.25304) 286 : cluster [DBG] pgmap v552: 425 pgs: 425 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 857 B/s rd, 1.2 KiB/s wr, 2 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:45:49.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:49 smithi077 bash[80681]: audit 2024-09-13T00:45:48.768751+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.77:0/1791209361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:49.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:49 smithi077 bash[83127]: cluster 2024-09-13T00:45:48.522473+0000 mgr.y (mgr.25304) 286 : cluster [DBG] pgmap v552: 425 pgs: 425 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 857 B/s rd, 1.2 KiB/s wr, 2 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:45:49.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:49 smithi077 bash[83127]: audit 2024-09-13T00:45:48.768751+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.77:0/1791209361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:49.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:49 smithi088 bash[68960]: cluster 2024-09-13T00:45:48.522473+0000 mgr.y (mgr.25304) 286 : cluster [DBG] pgmap v552: 425 pgs: 425 active+clean; 584 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 857 B/s rd, 1.2 KiB/s wr, 2 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:45:49.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:49 smithi088 bash[68960]: audit 2024-09-13T00:45:48.768751+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.77:0/1791209361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:51.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[80681]: audit 2024-09-13T00:45:49.982548+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:51.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[80681]: audit 2024-09-13T00:45:49.991785+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:51.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[80681]: audit 2024-09-13T00:45:49.993933+0000 mon.a (mon.0) 887 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:45:51.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[80681]: audit 2024-09-13T00:45:49.995444+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:45:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[80681]: audit 2024-09-13T00:45:50.005398+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[80681]: audit 2024-09-13T00:45:50.111401+0000 mon.a (mon.0) 890 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:45:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[80681]: audit 2024-09-13T00:45:50.115276+0000 mon.a (mon.0) 891 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[80681]: audit 2024-09-13T00:45:50.118164+0000 mon.a (mon.0) 892 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[80681]: audit 2024-09-13T00:45:50.120073+0000 mon.a (mon.0) 893 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[80681]: audit 2024-09-13T00:45:50.122224+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T00:45:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[80681]: audit 2024-09-13T00:45:50.122512+0000 mgr.y (mgr.25304) 287 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T00:45:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[80681]: cephadm 2024-09-13T00:45:50.124416+0000 mgr.y (mgr.25304) 288 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-09-13T00:45:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[80681]: cluster 2024-09-13T00:45:50.960532+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e635: 8 total, 8 up, 8 in 2024-09-13T00:45:51.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[83127]: audit 2024-09-13T00:45:49.982548+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[83127]: audit 2024-09-13T00:45:49.991785+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[83127]: audit 2024-09-13T00:45:49.993933+0000 mon.a (mon.0) 887 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:45:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[83127]: audit 2024-09-13T00:45:49.995444+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:45:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[83127]: audit 2024-09-13T00:45:50.005398+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[83127]: audit 2024-09-13T00:45:50.111401+0000 mon.a (mon.0) 890 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:45:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[83127]: audit 2024-09-13T00:45:50.115276+0000 mon.a (mon.0) 891 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[83127]: audit 2024-09-13T00:45:50.118164+0000 mon.a (mon.0) 892 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[83127]: audit 2024-09-13T00:45:50.120073+0000 mon.a (mon.0) 893 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[83127]: audit 2024-09-13T00:45:50.122224+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T00:45:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[83127]: audit 2024-09-13T00:45:50.122512+0000 mgr.y (mgr.25304) 287 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T00:45:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[83127]: cephadm 2024-09-13T00:45:50.124416+0000 mgr.y (mgr.25304) 288 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-09-13T00:45:51.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:50 smithi077 bash[83127]: cluster 2024-09-13T00:45:50.960532+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e635: 8 total, 8 up, 8 in 2024-09-13T00:45:51.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:50 smithi088 bash[68960]: audit 2024-09-13T00:45:49.982548+0000 mon.a (mon.0) 885 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:51.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:50 smithi088 bash[68960]: audit 2024-09-13T00:45:49.991785+0000 mon.a (mon.0) 886 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:51.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:50 smithi088 bash[68960]: audit 2024-09-13T00:45:49.993933+0000 mon.a (mon.0) 887 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:45:51.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:50 smithi088 bash[68960]: audit 2024-09-13T00:45:49.995444+0000 mon.a (mon.0) 888 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:45:51.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:50 smithi088 bash[68960]: audit 2024-09-13T00:45:50.005398+0000 mon.a (mon.0) 889 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:51.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:50 smithi088 bash[68960]: audit 2024-09-13T00:45:50.111401+0000 mon.a (mon.0) 890 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:45:51.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:50 smithi088 bash[68960]: audit 2024-09-13T00:45:50.115276+0000 mon.a (mon.0) 891 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:51.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:50 smithi088 bash[68960]: audit 2024-09-13T00:45:50.118164+0000 mon.a (mon.0) 892 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:51.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:50 smithi088 bash[68960]: audit 2024-09-13T00:45:50.120073+0000 mon.a (mon.0) 893 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:45:51.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:50 smithi088 bash[68960]: audit 2024-09-13T00:45:50.122224+0000 mon.a (mon.0) 894 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T00:45:51.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:50 smithi088 bash[68960]: audit 2024-09-13T00:45:50.122512+0000 mgr.y (mgr.25304) 287 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-13T00:45:51.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:50 smithi088 bash[68960]: cephadm 2024-09-13T00:45:50.124416+0000 mgr.y (mgr.25304) 288 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-09-13T00:45:51.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:50 smithi088 bash[68960]: cluster 2024-09-13T00:45:50.960532+0000 mon.a (mon.0) 895 : cluster [DBG] osdmap e635: 8 total, 8 up, 8 in 2024-09-13T00:45:52.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:52 smithi077 bash[80681]: cluster 2024-09-13T00:45:50.525903+0000 mgr.y (mgr.25304) 289 : cluster [DBG] pgmap v553: 425 pgs: 425 active+clean; 5.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.5 KiB/s rd, 736 KiB/s wr, 10 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:45:52.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:52 smithi077 bash[80681]: audit 2024-09-13T00:45:51.081200+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.77:0/2323395082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:52.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:52 smithi077 bash[83127]: cluster 2024-09-13T00:45:50.525903+0000 mgr.y (mgr.25304) 289 : cluster [DBG] pgmap v553: 425 pgs: 425 active+clean; 5.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.5 KiB/s rd, 736 KiB/s wr, 10 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:45:52.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:52 smithi077 bash[83127]: audit 2024-09-13T00:45:51.081200+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.77:0/2323395082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:52.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:52 smithi088 bash[68960]: cluster 2024-09-13T00:45:50.525903+0000 mgr.y (mgr.25304) 289 : cluster [DBG] pgmap v553: 425 pgs: 425 active+clean; 5.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.5 KiB/s rd, 736 KiB/s wr, 10 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:45:52.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:52 smithi088 bash[68960]: audit 2024-09-13T00:45:51.081200+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.77:0/2323395082' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:52.600 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:45:52 smithi088 bash[61791]: ts=2024-09-13T00:45:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:45:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:53 smithi077 bash[80681]: cluster 2024-09-13T00:45:52.527504+0000 mgr.y (mgr.25304) 290 : cluster [DBG] pgmap v555: 425 pgs: 425 active+clean; 5.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.6 KiB/s rd, 643 KiB/s wr, 7 op/s 2024-09-13T00:45:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:53 smithi077 bash[80681]: cephadm 2024-09-13T00:45:52.920578+0000 mgr.y (mgr.25304) 291 : cephadm [INF] Upgrade: Updating osd.3 2024-09-13T00:45:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:53 smithi077 bash[80681]: audit 2024-09-13T00:45:52.927475+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:53 smithi077 bash[80681]: audit 2024-09-13T00:45:52.928887+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T00:45:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:53 smithi077 bash[80681]: audit 2024-09-13T00:45:52.929871+0000 mon.a (mon.0) 898 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:45:54.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:53 smithi077 bash[80681]: cephadm 2024-09-13T00:45:52.936501+0000 mgr.y (mgr.25304) 292 : cephadm [INF] Deploying daemon osd.3 on smithi077 2024-09-13T00:45:54.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:53 smithi077 bash[80681]: audit 2024-09-13T00:45:53.030595+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:54.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:53 smithi077 bash[80681]: audit 2024-09-13T00:45:53.368848+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.77:0/4260859814' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:53 smithi077 bash[83127]: cluster 2024-09-13T00:45:52.527504+0000 mgr.y (mgr.25304) 290 : cluster [DBG] pgmap v555: 425 pgs: 425 active+clean; 5.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.6 KiB/s rd, 643 KiB/s wr, 7 op/s 2024-09-13T00:45:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:53 smithi077 bash[83127]: cephadm 2024-09-13T00:45:52.920578+0000 mgr.y (mgr.25304) 291 : cephadm [INF] Upgrade: Updating osd.3 2024-09-13T00:45:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:53 smithi077 bash[83127]: audit 2024-09-13T00:45:52.927475+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:53 smithi077 bash[83127]: audit 2024-09-13T00:45:52.928887+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T00:45:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:53 smithi077 bash[83127]: audit 2024-09-13T00:45:52.929871+0000 mon.a (mon.0) 898 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:45:54.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:53 smithi077 bash[83127]: cephadm 2024-09-13T00:45:52.936501+0000 mgr.y (mgr.25304) 292 : cephadm [INF] Deploying daemon osd.3 on smithi077 2024-09-13T00:45:54.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:53 smithi077 bash[83127]: audit 2024-09-13T00:45:53.030595+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:54.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:53 smithi077 bash[83127]: audit 2024-09-13T00:45:53.368848+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.77:0/4260859814' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:54.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:53 smithi088 bash[68960]: cluster 2024-09-13T00:45:52.527504+0000 mgr.y (mgr.25304) 290 : cluster [DBG] pgmap v555: 425 pgs: 425 active+clean; 5.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.6 KiB/s rd, 643 KiB/s wr, 7 op/s 2024-09-13T00:45:54.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:53 smithi088 bash[68960]: cephadm 2024-09-13T00:45:52.920578+0000 mgr.y (mgr.25304) 291 : cephadm [INF] Upgrade: Updating osd.3 2024-09-13T00:45:54.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:53 smithi088 bash[68960]: audit 2024-09-13T00:45:52.927475+0000 mon.a (mon.0) 896 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:54.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:53 smithi088 bash[68960]: audit 2024-09-13T00:45:52.928887+0000 mon.a (mon.0) 897 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-13T00:45:54.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:53 smithi088 bash[68960]: audit 2024-09-13T00:45:52.929871+0000 mon.a (mon.0) 898 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:45:54.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:53 smithi088 bash[68960]: cephadm 2024-09-13T00:45:52.936501+0000 mgr.y (mgr.25304) 292 : cephadm [INF] Deploying daemon osd.3 on smithi077 2024-09-13T00:45:54.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:53 smithi088 bash[68960]: audit 2024-09-13T00:45:53.030595+0000 mon.a (mon.0) 899 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:54.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:53 smithi088 bash[68960]: audit 2024-09-13T00:45:53.368848+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.77:0/4260859814' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:56.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:55 smithi077 bash[83127]: cluster 2024-09-13T00:45:54.529384+0000 mgr.y (mgr.25304) 293 : cluster [DBG] pgmap v556: 425 pgs: 425 active+clean; 11 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-13T00:45:56.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:55 smithi077 bash[83127]: audit 2024-09-13T00:45:55.658761+0000 mon.a (mon.0) 900 : audit [DBG] from='client.? 172.21.15.77:0/2023665875' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:56.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:55 smithi077 bash[80681]: cluster 2024-09-13T00:45:54.529384+0000 mgr.y (mgr.25304) 293 : cluster [DBG] pgmap v556: 425 pgs: 425 active+clean; 11 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-13T00:45:56.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:55 smithi077 bash[80681]: audit 2024-09-13T00:45:55.658761+0000 mon.a (mon.0) 900 : audit [DBG] from='client.? 172.21.15.77:0/2023665875' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:56.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:55 smithi088 bash[68960]: cluster 2024-09-13T00:45:54.529384+0000 mgr.y (mgr.25304) 293 : cluster [DBG] pgmap v556: 425 pgs: 425 active+clean; 11 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-13T00:45:56.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:55 smithi088 bash[68960]: audit 2024-09-13T00:45:55.658761+0000 mon.a (mon.0) 900 : audit [DBG] from='client.? 172.21.15.77:0/2023665875' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:56.851 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.RemoveFullTryDataPool (20376 ms) 2024-09-13T00:45:56.852 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestIO 2024-09-13T00:45:56.852 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:45:57.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:56 smithi077 bash[80681]: cluster 2024-09-13T00:45:55.956286+0000 mon.a (mon.0) 901 : cluster [WRN] pool 'test-librbd-smithi077-68798-24' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T00:45:57.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:56 smithi077 bash[80681]: cluster 2024-09-13T00:45:55.956887+0000 mon.a (mon.0) 902 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T00:45:57.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:56 smithi077 bash[80681]: cluster 2024-09-13T00:45:55.968052+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e636: 8 total, 8 up, 8 in 2024-09-13T00:45:57.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:56 smithi077 bash[83127]: cluster 2024-09-13T00:45:55.956286+0000 mon.a (mon.0) 901 : cluster [WRN] pool 'test-librbd-smithi077-68798-24' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T00:45:57.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:56 smithi077 bash[83127]: cluster 2024-09-13T00:45:55.956887+0000 mon.a (mon.0) 902 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T00:45:57.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:56 smithi077 bash[83127]: cluster 2024-09-13T00:45:55.968052+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e636: 8 total, 8 up, 8 in 2024-09-13T00:45:57.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:56 smithi088 bash[68960]: cluster 2024-09-13T00:45:55.956286+0000 mon.a (mon.0) 901 : cluster [WRN] pool 'test-librbd-smithi077-68798-24' is full (reached quota's max_bytes: 10 MiB) 2024-09-13T00:45:57.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:56 smithi088 bash[68960]: cluster 2024-09-13T00:45:55.956887+0000 mon.a (mon.0) 902 : cluster [WRN] Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-13T00:45:57.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:56 smithi088 bash[68960]: cluster 2024-09-13T00:45:55.968052+0000 mon.a (mon.0) 903 : cluster [DBG] osdmap e636: 8 total, 8 up, 8 in 2024-09-13T00:45:58.074 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:45:57 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:58.074 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:45:57 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:58.075 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:45:57 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:58.075 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:45:57 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:58.075 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:45:57 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:58.076 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:57 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:58.076 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:57 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:58.076 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:57 smithi077 bash[83127]: cluster 2024-09-13T00:45:56.530757+0000 mgr.y (mgr.25304) 294 : cluster [DBG] pgmap v558: 425 pgs: 425 active+clean; 11 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.7 KiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-13T00:45:58.076 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:57 smithi077 bash[83127]: audit 2024-09-13T00:45:57.958230+0000 mon.a (mon.0) 904 : audit [DBG] from='client.? 172.21.15.77:0/3404208163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:58.076 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:57 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:58.076 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:57 smithi077 bash[80681]: cluster 2024-09-13T00:45:56.530757+0000 mgr.y (mgr.25304) 294 : cluster [DBG] pgmap v558: 425 pgs: 425 active+clean; 11 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.7 KiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-13T00:45:58.077 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:57 smithi077 bash[80681]: audit 2024-09-13T00:45:57.958230+0000 mon.a (mon.0) 904 : audit [DBG] from='client.? 172.21.15.77:0/3404208163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:58.077 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:45:57 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:45:58.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:57 smithi088 bash[68960]: cluster 2024-09-13T00:45:56.530757+0000 mgr.y (mgr.25304) 294 : cluster [DBG] pgmap v558: 425 pgs: 425 active+clean; 11 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.7 KiB/s rd, 1.2 MiB/s wr, 8 op/s 2024-09-13T00:45:58.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:57 smithi088 bash[68960]: audit 2024-09-13T00:45:57.958230+0000 mon.a (mon.0) 904 : audit [DBG] from='client.? 172.21.15.77:0/3404208163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:45:58.525 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:45:58 smithi077 systemd[1]: Stopping Ceph osd.3 for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:45:58.525 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:45:58 smithi077 bash[51688]: debug 2024-09-13T00:45:58.196+0000 7fc40ac45640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:45:58.526 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:45:58 smithi077 bash[51688]: debug 2024-09-13T00:45:58.196+0000 7fc40ac45640 -1 osd.3 636 *** Got signal Terminated *** 2024-09-13T00:45:58.526 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:45:58 smithi077 bash[51688]: debug 2024-09-13T00:45:58.196+0000 7fc40ac45640 -1 osd.3 636 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T00:45:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:45:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:45:58] "GET /metrics HTTP/1.1" 200 60689 "" "Prometheus/2.43.0" 2024-09-13T00:45:59.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:58 smithi077 bash[83127]: cluster 2024-09-13T00:45:58.200975+0000 mon.a (mon.0) 905 : cluster [INF] osd.3 marked itself down and dead 2024-09-13T00:45:59.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:58 smithi077 bash[83127]: audit 2024-09-13T00:45:58.425700+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:59.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:58 smithi077 bash[83127]: audit 2024-09-13T00:45:58.427860+0000 mon.a (mon.0) 907 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:59.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:58 smithi077 bash[80681]: debug 2024-09-13T00:45:58.960+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:45:59.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:58 smithi077 bash[80681]: cluster 2024-09-13T00:45:58.200975+0000 mon.a (mon.0) 905 : cluster [INF] osd.3 marked itself down and dead 2024-09-13T00:45:59.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:58 smithi077 bash[80681]: audit 2024-09-13T00:45:58.425700+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:59.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:58 smithi077 bash[80681]: audit 2024-09-13T00:45:58.427860+0000 mon.a (mon.0) 907 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:45:59.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:58 smithi088 bash[68960]: cluster 2024-09-13T00:45:58.200975+0000 mon.a (mon.0) 905 : cluster [INF] osd.3 marked itself down and dead 2024-09-13T00:45:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:58 smithi088 bash[68960]: audit 2024-09-13T00:45:58.425700+0000 mon.a (mon.0) 906 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:45:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:58 smithi088 bash[68960]: audit 2024-09-13T00:45:58.427860+0000 mon.a (mon.0) 907 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[80681]: cluster 2024-09-13T00:45:58.531377+0000 mgr.y (mgr.25304) 295 : cluster [DBG] pgmap v559: 425 pgs: 425 active+clean; 11 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 639 B/s rd, 640 KiB/s wr, 1 op/s 2024-09-13T00:46:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[80681]: cluster 2024-09-13T00:45:58.966400+0000 mon.a (mon.0) 908 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T00:46:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[80681]: cluster 2024-09-13T00:45:58.966476+0000 mon.a (mon.0) 909 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:46:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[80681]: cluster 2024-09-13T00:45:58.984626+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e637: 8 total, 7 up, 8 in 2024-09-13T00:46:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[80681]: cluster 2024-09-13T00:45:58.987060+0000 mon.a (mon.0) 911 : cluster [INF] osd.3 marked itself dead as of e637 2024-09-13T00:46:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[80681]: audit 2024-09-13T00:45:59.002546+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.3", "id": [2, 3]}]: dispatch 2024-09-13T00:46:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[80681]: audit 2024-09-13T00:45:59.003148+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.c", "id": [2, 0]}]: dispatch 2024-09-13T00:46:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[80681]: audit 2024-09-13T00:45:59.003536+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.d", "id": [2, 5]}]: dispatch 2024-09-13T00:46:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[80681]: audit 2024-09-13T00:45:59.004273+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.11", "id": [6, 3]}]: dispatch 2024-09-13T00:46:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[80681]: audit 2024-09-13T00:45:59.004802+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.16", "id": [2, 3]}]: dispatch 2024-09-13T00:46:00.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[80681]: audit 2024-09-13T00:45:59.005265+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.1d", "id": [7, 3]}]: dispatch 2024-09-13T00:46:00.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[80681]: audit 2024-09-13T00:45:59.005693+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.3", "id": [6, 1]}]: dispatch 2024-09-13T00:46:00.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[80681]: audit 2024-09-13T00:45:59.006115+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.8", "id": [5, 1]}]: dispatch 2024-09-13T00:46:00.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[80681]: audit 2024-09-13T00:45:59.006540+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.11", "id": [2, 1]}]: dispatch 2024-09-13T00:46:00.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[80681]: audit 2024-09-13T00:45:59.006977+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.12", "id": [2, 4]}]: dispatch 2024-09-13T00:46:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[83127]: cluster 2024-09-13T00:45:58.531377+0000 mgr.y (mgr.25304) 295 : cluster [DBG] pgmap v559: 425 pgs: 425 active+clean; 11 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 639 B/s rd, 640 KiB/s wr, 1 op/s 2024-09-13T00:46:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[83127]: cluster 2024-09-13T00:45:58.966400+0000 mon.a (mon.0) 908 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T00:46:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[83127]: cluster 2024-09-13T00:45:58.966476+0000 mon.a (mon.0) 909 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:46:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[83127]: cluster 2024-09-13T00:45:58.984626+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e637: 8 total, 7 up, 8 in 2024-09-13T00:46:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[83127]: cluster 2024-09-13T00:45:58.987060+0000 mon.a (mon.0) 911 : cluster [INF] osd.3 marked itself dead as of e637 2024-09-13T00:46:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[83127]: audit 2024-09-13T00:45:59.002546+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.3", "id": [2, 3]}]: dispatch 2024-09-13T00:46:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[83127]: audit 2024-09-13T00:45:59.003148+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.c", "id": [2, 0]}]: dispatch 2024-09-13T00:46:00.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[83127]: audit 2024-09-13T00:45:59.003536+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.d", "id": [2, 5]}]: dispatch 2024-09-13T00:46:00.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[83127]: audit 2024-09-13T00:45:59.004273+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.11", "id": [6, 3]}]: dispatch 2024-09-13T00:46:00.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[83127]: audit 2024-09-13T00:45:59.004802+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.16", "id": [2, 3]}]: dispatch 2024-09-13T00:46:00.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[83127]: audit 2024-09-13T00:45:59.005265+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.1d", "id": [7, 3]}]: dispatch 2024-09-13T00:46:00.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[83127]: audit 2024-09-13T00:45:59.005693+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.3", "id": [6, 1]}]: dispatch 2024-09-13T00:46:00.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[83127]: audit 2024-09-13T00:45:59.006115+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.8", "id": [5, 1]}]: dispatch 2024-09-13T00:46:00.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[83127]: audit 2024-09-13T00:45:59.006540+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.11", "id": [2, 1]}]: dispatch 2024-09-13T00:46:00.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:45:59 smithi077 bash[83127]: audit 2024-09-13T00:45:59.006977+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.12", "id": [2, 4]}]: dispatch 2024-09-13T00:46:00.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:59 smithi088 bash[68960]: cluster 2024-09-13T00:45:58.531377+0000 mgr.y (mgr.25304) 295 : cluster [DBG] pgmap v559: 425 pgs: 425 active+clean; 11 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 639 B/s rd, 640 KiB/s wr, 1 op/s 2024-09-13T00:46:00.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:59 smithi088 bash[68960]: cluster 2024-09-13T00:45:58.966400+0000 mon.a (mon.0) 908 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T00:46:00.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:59 smithi088 bash[68960]: cluster 2024-09-13T00:45:58.966476+0000 mon.a (mon.0) 909 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:46:00.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:59 smithi088 bash[68960]: cluster 2024-09-13T00:45:58.984626+0000 mon.a (mon.0) 910 : cluster [DBG] osdmap e637: 8 total, 7 up, 8 in 2024-09-13T00:46:00.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:59 smithi088 bash[68960]: cluster 2024-09-13T00:45:58.987060+0000 mon.a (mon.0) 911 : cluster [INF] osd.3 marked itself dead as of e637 2024-09-13T00:46:00.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:59 smithi088 bash[68960]: audit 2024-09-13T00:45:59.002546+0000 mon.a (mon.0) 912 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.3", "id": [2, 3]}]: dispatch 2024-09-13T00:46:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:59 smithi088 bash[68960]: audit 2024-09-13T00:45:59.003148+0000 mon.a (mon.0) 913 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.c", "id": [2, 0]}]: dispatch 2024-09-13T00:46:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:59 smithi088 bash[68960]: audit 2024-09-13T00:45:59.003536+0000 mon.a (mon.0) 914 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.d", "id": [2, 5]}]: dispatch 2024-09-13T00:46:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:59 smithi088 bash[68960]: audit 2024-09-13T00:45:59.004273+0000 mon.a (mon.0) 915 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.11", "id": [6, 3]}]: dispatch 2024-09-13T00:46:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:59 smithi088 bash[68960]: audit 2024-09-13T00:45:59.004802+0000 mon.a (mon.0) 916 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.16", "id": [2, 3]}]: dispatch 2024-09-13T00:46:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:59 smithi088 bash[68960]: audit 2024-09-13T00:45:59.005265+0000 mon.a (mon.0) 917 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.1d", "id": [7, 3]}]: dispatch 2024-09-13T00:46:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:59 smithi088 bash[68960]: audit 2024-09-13T00:45:59.005693+0000 mon.a (mon.0) 918 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.3", "id": [6, 1]}]: dispatch 2024-09-13T00:46:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:59 smithi088 bash[68960]: audit 2024-09-13T00:45:59.006115+0000 mon.a (mon.0) 919 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.8", "id": [5, 1]}]: dispatch 2024-09-13T00:46:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:59 smithi088 bash[68960]: audit 2024-09-13T00:45:59.006540+0000 mon.a (mon.0) 920 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.11", "id": [2, 1]}]: dispatch 2024-09-13T00:46:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:45:59 smithi088 bash[68960]: audit 2024-09-13T00:45:59.006977+0000 mon.a (mon.0) 921 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.12", "id": [2, 4]}]: dispatch 2024-09-13T00:46:00.890 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[104315]: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-3 2024-09-13T00:46:01.278 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[104410]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-3 2024-09-13T00:46:01.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[83127]: audit 2024-09-13T00:45:59.983267+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.3", "id": [2, 3]}]': finished 2024-09-13T00:46:01.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[83127]: audit 2024-09-13T00:45:59.983372+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.c", "id": [2, 0]}]': finished 2024-09-13T00:46:01.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[83127]: audit 2024-09-13T00:45:59.983458+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.d", "id": [2, 5]}]': finished 2024-09-13T00:46:01.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[83127]: audit 2024-09-13T00:45:59.983527+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.11", "id": [6, 3]}]': finished 2024-09-13T00:46:01.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[83127]: audit 2024-09-13T00:45:59.983599+0000 mon.a (mon.0) 926 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.16", "id": [2, 3]}]': finished 2024-09-13T00:46:01.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[83127]: audit 2024-09-13T00:45:59.983672+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.1d", "id": [7, 3]}]': finished 2024-09-13T00:46:01.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[83127]: audit 2024-09-13T00:45:59.983744+0000 mon.a (mon.0) 928 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.3", "id": [6, 1]}]': finished 2024-09-13T00:46:01.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[83127]: audit 2024-09-13T00:45:59.983828+0000 mon.a (mon.0) 929 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.8", "id": [5, 1]}]': finished 2024-09-13T00:46:01.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[83127]: audit 2024-09-13T00:45:59.983894+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.11", "id": [2, 1]}]': finished 2024-09-13T00:46:01.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[83127]: audit 2024-09-13T00:45:59.983972+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.12", "id": [2, 4]}]': finished 2024-09-13T00:46:01.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[83127]: cluster 2024-09-13T00:45:59.984031+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e638: 8 total, 7 up, 8 in 2024-09-13T00:46:01.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[83127]: audit 2024-09-13T00:46:00.276080+0000 mon.a (mon.0) 933 : audit [DBG] from='client.? 172.21.15.77:0/801734392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:01.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[83127]: cluster 2024-09-13T00:46:00.967533+0000 mon.a (mon.0) 934 : cluster [DBG] osdmap e639: 8 total, 7 up, 8 in 2024-09-13T00:46:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[80681]: audit 2024-09-13T00:45:59.983267+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.3", "id": [2, 3]}]': finished 2024-09-13T00:46:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[80681]: audit 2024-09-13T00:45:59.983372+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.c", "id": [2, 0]}]': finished 2024-09-13T00:46:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[80681]: audit 2024-09-13T00:45:59.983458+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.d", "id": [2, 5]}]': finished 2024-09-13T00:46:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[80681]: audit 2024-09-13T00:45:59.983527+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.11", "id": [6, 3]}]': finished 2024-09-13T00:46:01.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[80681]: audit 2024-09-13T00:45:59.983599+0000 mon.a (mon.0) 926 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.16", "id": [2, 3]}]': finished 2024-09-13T00:46:01.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[80681]: audit 2024-09-13T00:45:59.983672+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.1d", "id": [7, 3]}]': finished 2024-09-13T00:46:01.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[80681]: audit 2024-09-13T00:45:59.983744+0000 mon.a (mon.0) 928 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.3", "id": [6, 1]}]': finished 2024-09-13T00:46:01.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[80681]: audit 2024-09-13T00:45:59.983828+0000 mon.a (mon.0) 929 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.8", "id": [5, 1]}]': finished 2024-09-13T00:46:01.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[80681]: audit 2024-09-13T00:45:59.983894+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.11", "id": [2, 1]}]': finished 2024-09-13T00:46:01.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[80681]: audit 2024-09-13T00:45:59.983972+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.12", "id": [2, 4]}]': finished 2024-09-13T00:46:01.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[80681]: cluster 2024-09-13T00:45:59.984031+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e638: 8 total, 7 up, 8 in 2024-09-13T00:46:01.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[80681]: audit 2024-09-13T00:46:00.276080+0000 mon.a (mon.0) 933 : audit [DBG] from='client.? 172.21.15.77:0/801734392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:01.281 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:00 smithi077 bash[80681]: cluster 2024-09-13T00:46:00.967533+0000 mon.a (mon.0) 934 : cluster [DBG] osdmap e639: 8 total, 7 up, 8 in 2024-09-13T00:46:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:00 smithi088 bash[68960]: audit 2024-09-13T00:45:59.983267+0000 mon.a (mon.0) 922 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.3", "id": [2, 3]}]': finished 2024-09-13T00:46:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:00 smithi088 bash[68960]: audit 2024-09-13T00:45:59.983372+0000 mon.a (mon.0) 923 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.c", "id": [2, 0]}]': finished 2024-09-13T00:46:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:00 smithi088 bash[68960]: audit 2024-09-13T00:45:59.983458+0000 mon.a (mon.0) 924 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.d", "id": [2, 5]}]': finished 2024-09-13T00:46:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:00 smithi088 bash[68960]: audit 2024-09-13T00:45:59.983527+0000 mon.a (mon.0) 925 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.11", "id": [6, 3]}]': finished 2024-09-13T00:46:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:00 smithi088 bash[68960]: audit 2024-09-13T00:45:59.983599+0000 mon.a (mon.0) 926 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.16", "id": [2, 3]}]': finished 2024-09-13T00:46:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:00 smithi088 bash[68960]: audit 2024-09-13T00:45:59.983672+0000 mon.a (mon.0) 927 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "19.1d", "id": [7, 3]}]': finished 2024-09-13T00:46:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:00 smithi088 bash[68960]: audit 2024-09-13T00:45:59.983744+0000 mon.a (mon.0) 928 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.3", "id": [6, 1]}]': finished 2024-09-13T00:46:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:00 smithi088 bash[68960]: audit 2024-09-13T00:45:59.983828+0000 mon.a (mon.0) 929 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.8", "id": [5, 1]}]': finished 2024-09-13T00:46:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:00 smithi088 bash[68960]: audit 2024-09-13T00:45:59.983894+0000 mon.a (mon.0) 930 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.11", "id": [2, 1]}]': finished 2024-09-13T00:46:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:00 smithi088 bash[68960]: audit 2024-09-13T00:45:59.983972+0000 mon.a (mon.0) 931 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "27.12", "id": [2, 4]}]': finished 2024-09-13T00:46:01.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:00 smithi088 bash[68960]: cluster 2024-09-13T00:45:59.984031+0000 mon.a (mon.0) 932 : cluster [DBG] osdmap e638: 8 total, 7 up, 8 in 2024-09-13T00:46:01.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:00 smithi088 bash[68960]: audit 2024-09-13T00:46:00.276080+0000 mon.a (mon.0) 933 : audit [DBG] from='client.? 172.21.15.77:0/801734392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:01.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:00 smithi088 bash[68960]: cluster 2024-09-13T00:46:00.967533+0000 mon.a (mon.0) 934 : cluster [DBG] osdmap e639: 8 total, 7 up, 8 in 2024-09-13T00:46:01.604 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:01.605 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:01.605 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:01.605 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:01.605 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:01.605 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.605 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:01.605 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.605 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:01.605 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:01.605 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.606 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:01.606 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.606 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:01.606 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:01.606 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.606 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:01.606 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.606 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:01.606 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:01.606 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.606 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:01.606 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.607 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:01.607 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:01.607 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.607 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:01.607 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.607 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:01.607 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:01.607 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:01.607 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:01.607 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:01.607 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:01.608 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:01.608 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:01.608 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:01.608 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:01.608 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:01.608 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:01.608 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.608 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:01.608 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.608 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:01.608 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:01.608 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.609 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:01.609 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.609 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:01.609 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:01.609 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.609 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:01.609 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.609 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:01.609 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:01.609 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.609 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:01.609 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.609 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:01.610 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:01.610 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.610 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:01.610 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.610 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:01.610 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:01.610 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:01.610 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:01.610 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:01.610 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:01.610 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:01.610 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.610 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:01.610 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:01.610 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:01.611 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.611 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:01.611 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:01.611 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:01.611 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.611 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:01.611 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:01.611 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:01.611 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.611 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:01.611 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:01.611 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:01.612 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.612 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:01.612 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:01.612 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:01.612 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 512~512 = 512 2024-09-13T00:46:01.612 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.612 INFO:tasks.workunit.client.0.smithi077.stdout:aio discard: 1536~512 = 0 2024-09-13T00:46:01.612 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:01.612 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:01.612 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:01.612 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:01.612 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:01.612 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 0 2024-09-13T00:46:01.613 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.613 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.613 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 0 2024-09-13T00:46:01.613 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.613 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.613 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 16384 2024-09-13T00:46:01.613 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.613 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.613 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 16384 2024-09-13T00:46:01.613 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.613 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.613 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:01.613 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:01.613 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 49152 2024-09-13T00:46:01.614 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.614 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.614 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 49152 2024-09-13T00:46:01.614 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.614 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.614 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 65536 2024-09-13T00:46:01.614 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.614 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.614 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 65536 2024-09-13T00:46:01.614 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.614 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.614 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:01.614 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:01.614 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 98304 2024-09-13T00:46:01.615 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.615 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.615 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 98304 2024-09-13T00:46:01.615 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.615 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.615 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 114688 2024-09-13T00:46:01.615 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.615 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.615 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 114688 2024-09-13T00:46:01.615 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.615 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.615 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:01.615 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:01.616 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 147456 2024-09-13T00:46:01.616 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.616 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.616 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 147456 2024-09-13T00:46:01.616 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.616 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.616 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 163840 2024-09-13T00:46:01.616 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.616 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.616 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 163840 2024-09-13T00:46:01.616 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.617 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.617 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:01.617 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:01.617 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 196608 2024-09-13T00:46:01.617 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.617 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.617 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 196608 2024-09-13T00:46:01.617 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.617 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.617 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 212992 2024-09-13T00:46:01.617 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.617 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.617 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 212992 2024-09-13T00:46:01.618 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.618 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.618 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:01.618 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:01.618 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.618 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:01.618 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.618 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:01.618 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:01.618 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.618 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.618 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.619 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:01.619 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.619 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:01.619 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:01.619 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.619 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.619 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.619 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:01.619 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.619 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:01.619 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:01.619 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.619 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.620 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.620 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:01.620 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.620 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:01.620 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:01.620 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.620 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.620 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.620 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:01.620 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:01.620 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.620 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:01.621 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:01.621 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.621 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:01.621 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.621 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:01.621 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:01.621 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.621 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.621 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:01.621 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:01.621 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:01.621 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:01.621 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:01.622 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:01.622 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:01.622 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:02.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:01 smithi077 bash[83127]: cluster 2024-09-13T00:46:00.532874+0000 mgr.y (mgr.25304) 296 : cluster [DBG] pgmap v562: 425 pgs: 160 peering, 6 stale+active+clean, 259 active+clean; 585 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 453 KiB/s wr, 359 op/s 2024-09-13T00:46:02.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:01 smithi077 bash[83127]: cluster 2024-09-13T00:46:00.977975+0000 mon.a (mon.0) 935 : cluster [WRN] Health check failed: Reduced data availability: 18 pgs inactive, 37 pgs peering (PG_AVAILABILITY) 2024-09-13T00:46:02.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:01 smithi077 bash[83127]: cluster 2024-09-13T00:46:01.980795+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e640: 8 total, 7 up, 8 in 2024-09-13T00:46:02.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:01 smithi077 bash[80681]: cluster 2024-09-13T00:46:00.532874+0000 mgr.y (mgr.25304) 296 : cluster [DBG] pgmap v562: 425 pgs: 160 peering, 6 stale+active+clean, 259 active+clean; 585 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 453 KiB/s wr, 359 op/s 2024-09-13T00:46:02.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:01 smithi077 bash[80681]: cluster 2024-09-13T00:46:00.977975+0000 mon.a (mon.0) 935 : cluster [WRN] Health check failed: Reduced data availability: 18 pgs inactive, 37 pgs peering (PG_AVAILABILITY) 2024-09-13T00:46:02.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:01 smithi077 bash[80681]: cluster 2024-09-13T00:46:01.980795+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e640: 8 total, 7 up, 8 in 2024-09-13T00:46:02.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:01 smithi088 bash[68960]: cluster 2024-09-13T00:46:00.532874+0000 mgr.y (mgr.25304) 296 : cluster [DBG] pgmap v562: 425 pgs: 160 peering, 6 stale+active+clean, 259 active+clean; 585 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 453 KiB/s wr, 359 op/s 2024-09-13T00:46:02.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:01 smithi088 bash[68960]: cluster 2024-09-13T00:46:00.977975+0000 mon.a (mon.0) 935 : cluster [WRN] Health check failed: Reduced data availability: 18 pgs inactive, 37 pgs peering (PG_AVAILABILITY) 2024-09-13T00:46:02.312 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:01 smithi088 bash[68960]: cluster 2024-09-13T00:46:01.980795+0000 mon.a (mon.0) 936 : cluster [DBG] osdmap e640: 8 total, 7 up, 8 in 2024-09-13T00:46:02.600 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:46:02 smithi088 bash[61791]: ts=2024-09-13T00:46:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:46:03.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:03 smithi077 bash[80681]: audit 2024-09-13T00:46:02.594357+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.77:0/3636587413' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:03.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:03 smithi077 bash[83127]: audit 2024-09-13T00:46:02.594357+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.77:0/3636587413' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:03.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:03 smithi088 bash[68960]: audit 2024-09-13T00:46:02.594357+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.77:0/3636587413' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:03.923 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:46:03 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:03.923 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:46:03 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:03.923 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:46:03 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:03.924 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:46:03 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:03.924 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:03 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:03.924 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:03 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-13T00:46:03.924 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:03 smithi077 systemd[1]: Stopped Ceph osd.3 for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:46:03.924 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:03 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:03.925 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:03 smithi077 systemd[1]: Started Ceph osd.3 for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:46:03.925 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 00:46:03 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:03.925 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:46:03 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:03.925 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:03 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:04.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:04 smithi077 bash[80681]: cluster 2024-09-13T00:46:02.533662+0000 mgr.y (mgr.25304) 297 : cluster [DBG] pgmap v565: 425 pgs: 2 remapped+peering, 18 active+undersized, 159 peering, 5 active+undersized+degraded, 241 active+clean; 585 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 273 KiB/s rd, 680 KiB/s wr, 542 op/s; 8/522 objects degraded (1.533%) 2024-09-13T00:46:04.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:04 smithi077 bash[80681]: cluster 2024-09-13T00:46:02.990272+0000 mon.a (mon.0) 937 : cluster [WRN] Health check failed: Degraded data redundancy: 8/522 objects degraded (1.533%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:04.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:04 smithi077 bash[80681]: audit 2024-09-13T00:46:03.963309+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:04.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:04 smithi077 bash[80681]: audit 2024-09-13T00:46:03.969349+0000 mon.a (mon.0) 939 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:04.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:04 smithi077 bash[80681]: audit 2024-09-13T00:46:03.970457+0000 mon.a (mon.0) 940 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:46:04.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:04 smithi077 bash[83127]: cluster 2024-09-13T00:46:02.533662+0000 mgr.y (mgr.25304) 297 : cluster [DBG] pgmap v565: 425 pgs: 2 remapped+peering, 18 active+undersized, 159 peering, 5 active+undersized+degraded, 241 active+clean; 585 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 273 KiB/s rd, 680 KiB/s wr, 542 op/s; 8/522 objects degraded (1.533%) 2024-09-13T00:46:04.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:04 smithi077 bash[83127]: cluster 2024-09-13T00:46:02.990272+0000 mon.a (mon.0) 937 : cluster [WRN] Health check failed: Degraded data redundancy: 8/522 objects degraded (1.533%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:04.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:04 smithi077 bash[83127]: audit 2024-09-13T00:46:03.963309+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:04.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:04 smithi077 bash[83127]: audit 2024-09-13T00:46:03.969349+0000 mon.a (mon.0) 939 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:04.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:04 smithi077 bash[83127]: audit 2024-09-13T00:46:03.970457+0000 mon.a (mon.0) 940 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:46:04.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:04 smithi088 bash[68960]: cluster 2024-09-13T00:46:02.533662+0000 mgr.y (mgr.25304) 297 : cluster [DBG] pgmap v565: 425 pgs: 2 remapped+peering, 18 active+undersized, 159 peering, 5 active+undersized+degraded, 241 active+clean; 585 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 273 KiB/s rd, 680 KiB/s wr, 542 op/s; 8/522 objects degraded (1.533%) 2024-09-13T00:46:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:04 smithi088 bash[68960]: cluster 2024-09-13T00:46:02.990272+0000 mon.a (mon.0) 937 : cluster [WRN] Health check failed: Degraded data redundancy: 8/522 objects degraded (1.533%), 5 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:04 smithi088 bash[68960]: audit 2024-09-13T00:46:03.963309+0000 mon.a (mon.0) 938 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:04 smithi088 bash[68960]: audit 2024-09-13T00:46:03.969349+0000 mon.a (mon.0) 939 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:04 smithi088 bash[68960]: audit 2024-09-13T00:46:03.970457+0000 mon.a (mon.0) 940 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:46:05.277 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:05 smithi077 bash[83127]: audit 2024-09-13T00:46:04.883919+0000 mon.a (mon.0) 941 : audit [DBG] from='client.? 172.21.15.77:0/3595530967' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:05.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:05 smithi077 bash[80681]: audit 2024-09-13T00:46:04.883919+0000 mon.a (mon.0) 941 : audit [DBG] from='client.? 172.21.15.77:0/3595530967' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:05.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:05 smithi088 bash[68960]: audit 2024-09-13T00:46:04.883919+0000 mon.a (mon.0) 941 : audit [DBG] from='client.? 172.21.15.77:0/3595530967' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:05.717 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.717 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:05.717 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:05.718 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:05.718 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:05.718 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:05.718 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.718 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.718 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:05.718 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:05.718 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:05.718 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:05.718 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:05.718 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.719 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.719 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:05.719 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.719 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.719 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:05.719 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.719 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.719 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:05.719 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:05.720 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:05.720 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:05.720 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:05.720 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.720 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.720 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:05.720 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:05.720 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:05.720 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:05.720 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:05.720 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.720 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.720 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:05.720 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:05.721 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:05.721 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:05.721 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:05.721 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.721 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.721 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:05.721 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:05.721 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:05.721 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:05.721 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:05.721 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.721 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.721 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:05.721 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.722 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.722 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:05.722 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.722 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.722 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:05.722 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:05.722 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:05.722 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:05.722 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:05.722 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.722 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.722 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:05.722 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:05.722 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:05.723 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:05.723 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:05.723 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.723 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.723 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:05.723 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:05.723 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:05.723 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:05.723 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:05.723 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.723 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.723 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:05.723 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:05.723 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:05.723 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:05.724 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:05.724 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.724 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.724 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:05.724 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.724 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.724 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:05.724 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.724 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.724 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:05.724 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:05.724 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:05.724 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:05.724 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:05.725 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.725 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.725 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:05.725 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:05.725 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:05.725 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:05.725 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:05.725 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.725 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.725 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:05.725 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:05.725 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:05.725 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:05.726 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:05.726 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.726 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.726 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:05.726 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:05.726 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:05.726 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:05.726 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:05.726 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.726 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.726 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:05.726 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:05.726 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:05.726 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:05.726 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:05.727 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:05.727 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:05.727 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:05.727 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestIO (8866 ms) 2024-09-13T00:46:05.727 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS1 2024-09-13T00:46:05.727 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:06.075 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:06 smithi077 bash[80681]: cluster 2024-09-13T00:46:04.536819+0000 mgr.y (mgr.25304) 298 : cluster [DBG] pgmap v566: 425 pgs: 7 active+clean+remapped, 2 remapped+peering, 113 active+undersized, 22 peering, 43 active+undersized+degraded, 238 active+clean; 782 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 627 KiB/s rd, 712 KiB/s wr, 1.25k op/s; 63/522 objects degraded (12.069%); 2/522 objects misplaced (0.383%); 0 B/s, 0 objects/s recovering 2024-09-13T00:46:06.075 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:06 smithi077 bash[80681]: cluster 2024-09-13T00:46:05.959848+0000 mon.a (mon.0) 942 : cluster [INF] pool 'test-librbd-smithi077-68798-24' no longer out of quota; removing NO_QUOTA flag 2024-09-13T00:46:06.075 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:06 smithi077 bash[80681]: cluster 2024-09-13T00:46:05.960174+0000 mon.a (mon.0) 943 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T00:46:06.075 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:06 smithi077 bash[80681]: cluster 2024-09-13T00:46:05.971775+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e641: 8 total, 7 up, 8 in 2024-09-13T00:46:06.075 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:05 smithi077 bash[104580]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T00:46:06.075 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:05 smithi077 bash[104580]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-13T00:46:06.076 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:05 smithi077 bash[104580]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-13T00:46:06.076 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:05 smithi077 bash[104580]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-13T00:46:06.076 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:05 smithi077 bash[104580]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-13T00:46:06.076 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:05 smithi077 bash[104580]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-13T00:46:06.076 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:05 smithi077 bash[104580]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-13T00:46:06.076 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:06 smithi077 bash[83127]: cluster 2024-09-13T00:46:04.536819+0000 mgr.y (mgr.25304) 298 : cluster [DBG] pgmap v566: 425 pgs: 7 active+clean+remapped, 2 remapped+peering, 113 active+undersized, 22 peering, 43 active+undersized+degraded, 238 active+clean; 782 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 627 KiB/s rd, 712 KiB/s wr, 1.25k op/s; 63/522 objects degraded (12.069%); 2/522 objects misplaced (0.383%); 0 B/s, 0 objects/s recovering 2024-09-13T00:46:06.076 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:06 smithi077 bash[83127]: cluster 2024-09-13T00:46:05.959848+0000 mon.a (mon.0) 942 : cluster [INF] pool 'test-librbd-smithi077-68798-24' no longer out of quota; removing NO_QUOTA flag 2024-09-13T00:46:06.076 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:06 smithi077 bash[83127]: cluster 2024-09-13T00:46:05.960174+0000 mon.a (mon.0) 943 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T00:46:06.076 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:06 smithi077 bash[83127]: cluster 2024-09-13T00:46:05.971775+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e641: 8 total, 7 up, 8 in 2024-09-13T00:46:06.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:06 smithi088 bash[68960]: cluster 2024-09-13T00:46:04.536819+0000 mgr.y (mgr.25304) 298 : cluster [DBG] pgmap v566: 425 pgs: 7 active+clean+remapped, 2 remapped+peering, 113 active+undersized, 22 peering, 43 active+undersized+degraded, 238 active+clean; 782 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 627 KiB/s rd, 712 KiB/s wr, 1.25k op/s; 63/522 objects degraded (12.069%); 2/522 objects misplaced (0.383%); 0 B/s, 0 objects/s recovering 2024-09-13T00:46:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:06 smithi088 bash[68960]: cluster 2024-09-13T00:46:05.959848+0000 mon.a (mon.0) 942 : cluster [INF] pool 'test-librbd-smithi077-68798-24' no longer out of quota; removing NO_QUOTA flag 2024-09-13T00:46:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:06 smithi088 bash[68960]: cluster 2024-09-13T00:46:05.960174+0000 mon.a (mon.0) 943 : cluster [INF] Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-13T00:46:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:06 smithi088 bash[68960]: cluster 2024-09-13T00:46:05.971775+0000 mon.a (mon.0) 944 : cluster [DBG] osdmap e641: 8 total, 7 up, 8 in 2024-09-13T00:46:07.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:07 smithi088 bash[68960]: cluster 2024-09-13T00:46:07.011985+0000 mon.a (mon.0) 945 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T00:46:07.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:07 smithi077 bash[80681]: cluster 2024-09-13T00:46:07.011985+0000 mon.a (mon.0) 945 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T00:46:07.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:07 smithi077 bash[83127]: cluster 2024-09-13T00:46:07.011985+0000 mon.a (mon.0) 945 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-13T00:46:08.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:08 smithi077 bash[80681]: cluster 2024-09-13T00:46:06.538739+0000 mgr.y (mgr.25304) 299 : cluster [DBG] pgmap v568: 425 pgs: 9 active+clean+remapped, 131 active+undersized, 47 active+undersized+degraded, 238 active+clean; 782 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 399 KiB/s rd, 205 KiB/s wr, 798 op/s; 72/525 objects degraded (13.714%); 2/525 objects misplaced (0.381%); 0 B/s, 0 objects/s recovering 2024-09-13T00:46:08.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:08 smithi077 bash[80681]: audit 2024-09-13T00:46:07.170538+0000 mon.a (mon.0) 946 : audit [DBG] from='client.? 172.21.15.77:0/3132716412' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:08.278 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:08 smithi077 bash[104829]: debug 2024-09-13T00:46:08.004+0000 7f86321d6640 1 -- 172.21.15.77:0/3310008245 <== mon.2 v2:172.21.15.77:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560db83351e0 con 0x560db82f5c00 2024-09-13T00:46:08.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:08 smithi077 bash[83127]: cluster 2024-09-13T00:46:06.538739+0000 mgr.y (mgr.25304) 299 : cluster [DBG] pgmap v568: 425 pgs: 9 active+clean+remapped, 131 active+undersized, 47 active+undersized+degraded, 238 active+clean; 782 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 399 KiB/s rd, 205 KiB/s wr, 798 op/s; 72/525 objects degraded (13.714%); 2/525 objects misplaced (0.381%); 0 B/s, 0 objects/s recovering 2024-09-13T00:46:08.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:08 smithi077 bash[83127]: audit 2024-09-13T00:46:07.170538+0000 mon.a (mon.0) 946 : audit [DBG] from='client.? 172.21.15.77:0/3132716412' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:08.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:08 smithi088 bash[68960]: cluster 2024-09-13T00:46:06.538739+0000 mgr.y (mgr.25304) 299 : cluster [DBG] pgmap v568: 425 pgs: 9 active+clean+remapped, 131 active+undersized, 47 active+undersized+degraded, 238 active+clean; 782 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 399 KiB/s rd, 205 KiB/s wr, 798 op/s; 72/525 objects degraded (13.714%); 2/525 objects misplaced (0.381%); 0 B/s, 0 objects/s recovering 2024-09-13T00:46:08.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:08 smithi088 bash[68960]: audit 2024-09-13T00:46:07.170538+0000 mon.a (mon.0) 946 : audit [DBG] from='client.? 172.21.15.77:0/3132716412' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:08.777 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:08 smithi077 bash[104829]: debug 2024-09-13T00:46:08.552+0000 7f8634a3c640 -1 Falling back to public interface 2024-09-13T00:46:08.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:46:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:46:08] "GET /metrics HTTP/1.1" 200 60689 "" "Prometheus/2.43.0" 2024-09-13T00:46:10.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:10 smithi088 bash[68960]: cluster 2024-09-13T00:46:08.540118+0000 mgr.y (mgr.25304) 300 : cluster [DBG] pgmap v569: 425 pgs: 9 active+clean+remapped, 131 active+undersized, 47 active+undersized+degraded, 238 active+clean; 782 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 316 KiB/s rd, 163 KiB/s wr, 633 op/s; 72/525 objects degraded (13.714%); 2/525 objects misplaced (0.381%); 0 B/s, 0 objects/s recovering 2024-09-13T00:46:10.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:10 smithi088 bash[68960]: audit 2024-09-13T00:46:09.456519+0000 mon.a (mon.0) 947 : audit [DBG] from='client.? 172.21.15.77:0/2072509897' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:10.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:10 smithi077 bash[80681]: cluster 2024-09-13T00:46:08.540118+0000 mgr.y (mgr.25304) 300 : cluster [DBG] pgmap v569: 425 pgs: 9 active+clean+remapped, 131 active+undersized, 47 active+undersized+degraded, 238 active+clean; 782 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 316 KiB/s rd, 163 KiB/s wr, 633 op/s; 72/525 objects degraded (13.714%); 2/525 objects misplaced (0.381%); 0 B/s, 0 objects/s recovering 2024-09-13T00:46:10.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:10 smithi077 bash[80681]: audit 2024-09-13T00:46:09.456519+0000 mon.a (mon.0) 947 : audit [DBG] from='client.? 172.21.15.77:0/2072509897' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:10.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:10 smithi077 bash[83127]: cluster 2024-09-13T00:46:08.540118+0000 mgr.y (mgr.25304) 300 : cluster [DBG] pgmap v569: 425 pgs: 9 active+clean+remapped, 131 active+undersized, 47 active+undersized+degraded, 238 active+clean; 782 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 316 KiB/s rd, 163 KiB/s wr, 633 op/s; 72/525 objects degraded (13.714%); 2/525 objects misplaced (0.381%); 0 B/s, 0 objects/s recovering 2024-09-13T00:46:10.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:10 smithi077 bash[83127]: audit 2024-09-13T00:46:09.456519+0000 mon.a (mon.0) 947 : audit [DBG] from='client.? 172.21.15.77:0/2072509897' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:12.028 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[104829]: debug 2024-09-13T00:46:11.736+0000 7f8634a3c640 -1 osd.3 637 log_to_monitors true 2024-09-13T00:46:12.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[80681]: cluster 2024-09-13T00:46:10.541733+0000 mgr.y (mgr.25304) 301 : cluster [DBG] pgmap v570: 425 pgs: 9 active+clean+remapped, 131 active+undersized, 47 active+undersized+degraded, 238 active+clean; 2.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 350 KiB/s rd, 175 KiB/s wr, 727 op/s; 74/534 objects degraded (13.858%); 2/534 objects misplaced (0.375%); 0 B/s, 0 objects/s recovering 2024-09-13T00:46:12.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[80681]: audit 2024-09-13T00:46:10.903604+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:12.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[80681]: audit 2024-09-13T00:46:10.924385+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:12.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[80681]: cluster 2024-09-13T00:46:10.962381+0000 mon.a (mon.0) 950 : cluster [WRN] Health check update: Degraded data redundancy: 72/525 objects degraded (13.714%), 47 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:12.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[80681]: audit 2024-09-13T00:46:11.639555+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:12.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[80681]: audit 2024-09-13T00:46:11.645213+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:12.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[80681]: audit 2024-09-13T00:46:11.743311+0000 mon.a (mon.0) 953 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2452864905,v1:172.21.15.77:6827/2452864905]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T00:46:12.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[80681]: audit 2024-09-13T00:46:11.762629+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.77:0/500500558' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[83127]: cluster 2024-09-13T00:46:10.541733+0000 mgr.y (mgr.25304) 301 : cluster [DBG] pgmap v570: 425 pgs: 9 active+clean+remapped, 131 active+undersized, 47 active+undersized+degraded, 238 active+clean; 2.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 350 KiB/s rd, 175 KiB/s wr, 727 op/s; 74/534 objects degraded (13.858%); 2/534 objects misplaced (0.375%); 0 B/s, 0 objects/s recovering 2024-09-13T00:46:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[83127]: audit 2024-09-13T00:46:10.903604+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[83127]: audit 2024-09-13T00:46:10.924385+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[83127]: cluster 2024-09-13T00:46:10.962381+0000 mon.a (mon.0) 950 : cluster [WRN] Health check update: Degraded data redundancy: 72/525 objects degraded (13.714%), 47 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[83127]: audit 2024-09-13T00:46:11.639555+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[83127]: audit 2024-09-13T00:46:11.645213+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[83127]: audit 2024-09-13T00:46:11.743311+0000 mon.a (mon.0) 953 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2452864905,v1:172.21.15.77:6827/2452864905]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T00:46:12.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:11 smithi077 bash[83127]: audit 2024-09-13T00:46:11.762629+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.77:0/500500558' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:12.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:11 smithi088 bash[68960]: cluster 2024-09-13T00:46:10.541733+0000 mgr.y (mgr.25304) 301 : cluster [DBG] pgmap v570: 425 pgs: 9 active+clean+remapped, 131 active+undersized, 47 active+undersized+degraded, 238 active+clean; 2.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 350 KiB/s rd, 175 KiB/s wr, 727 op/s; 74/534 objects degraded (13.858%); 2/534 objects misplaced (0.375%); 0 B/s, 0 objects/s recovering 2024-09-13T00:46:12.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:11 smithi088 bash[68960]: audit 2024-09-13T00:46:10.903604+0000 mon.a (mon.0) 948 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:12.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:11 smithi088 bash[68960]: audit 2024-09-13T00:46:10.924385+0000 mon.a (mon.0) 949 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:12.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:11 smithi088 bash[68960]: cluster 2024-09-13T00:46:10.962381+0000 mon.a (mon.0) 950 : cluster [WRN] Health check update: Degraded data redundancy: 72/525 objects degraded (13.714%), 47 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:12.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:11 smithi088 bash[68960]: audit 2024-09-13T00:46:11.639555+0000 mon.a (mon.0) 951 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:12.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:11 smithi088 bash[68960]: audit 2024-09-13T00:46:11.645213+0000 mon.a (mon.0) 952 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:12.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:11 smithi088 bash[68960]: audit 2024-09-13T00:46:11.743311+0000 mon.a (mon.0) 953 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2452864905,v1:172.21.15.77:6827/2452864905]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-13T00:46:12.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:11 smithi088 bash[68960]: audit 2024-09-13T00:46:11.762629+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.77:0/500500558' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:12.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:46:12 smithi088 bash[61791]: ts=2024-09-13T00:46:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:46:13.527 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[104829]: debug 2024-09-13T00:46:13.235+0000 7f86287de640 -1 osd.3 637 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T00:46:14.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[83127]: cluster 2024-09-13T00:46:12.543623+0000 mgr.y (mgr.25304) 302 : cluster [DBG] pgmap v571: 425 pgs: 9 active+clean+remapped, 131 active+undersized, 47 active+undersized+degraded, 238 active+clean; 2.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 301 KiB/s rd, 150 KiB/s wr, 625 op/s; 74/534 objects degraded (13.858%); 2/534 objects misplaced (0.375%); 0 B/s, 0 objects/s recovering 2024-09-13T00:46:14.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[83127]: audit 2024-09-13T00:46:12.654753+0000 mon.a (mon.0) 954 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2452864905,v1:172.21.15.77:6827/2452864905]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T00:46:14.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[83127]: cluster 2024-09-13T00:46:12.655174+0000 mon.a (mon.0) 955 : cluster [DBG] osdmap e642: 8 total, 7 up, 8 in 2024-09-13T00:46:14.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[83127]: audit 2024-09-13T00:46:12.658058+0000 mon.a (mon.0) 956 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2452864905,v1:172.21.15.77:6827/2452864905]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:46:14.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[83127]: audit 2024-09-13T00:46:13.072521+0000 mon.a (mon.0) 957 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:14.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[83127]: audit 2024-09-13T00:46:13.219274+0000 mon.a (mon.0) 958 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2452864905,v1:172.21.15.77:6827/2452864905]' entity='osd.3' 2024-09-13T00:46:14.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[83127]: audit 2024-09-13T00:46:13.425126+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:14.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[83127]: audit 2024-09-13T00:46:13.427306+0000 mon.a (mon.0) 960 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:14.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[80681]: cluster 2024-09-13T00:46:12.543623+0000 mgr.y (mgr.25304) 302 : cluster [DBG] pgmap v571: 425 pgs: 9 active+clean+remapped, 131 active+undersized, 47 active+undersized+degraded, 238 active+clean; 2.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 301 KiB/s rd, 150 KiB/s wr, 625 op/s; 74/534 objects degraded (13.858%); 2/534 objects misplaced (0.375%); 0 B/s, 0 objects/s recovering 2024-09-13T00:46:14.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[80681]: audit 2024-09-13T00:46:12.654753+0000 mon.a (mon.0) 954 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2452864905,v1:172.21.15.77:6827/2452864905]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T00:46:14.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[80681]: cluster 2024-09-13T00:46:12.655174+0000 mon.a (mon.0) 955 : cluster [DBG] osdmap e642: 8 total, 7 up, 8 in 2024-09-13T00:46:14.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[80681]: audit 2024-09-13T00:46:12.658058+0000 mon.a (mon.0) 956 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2452864905,v1:172.21.15.77:6827/2452864905]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:46:14.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[80681]: audit 2024-09-13T00:46:13.072521+0000 mon.a (mon.0) 957 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:14.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[80681]: audit 2024-09-13T00:46:13.219274+0000 mon.a (mon.0) 958 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2452864905,v1:172.21.15.77:6827/2452864905]' entity='osd.3' 2024-09-13T00:46:14.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[80681]: audit 2024-09-13T00:46:13.425126+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:14.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:13 smithi077 bash[80681]: audit 2024-09-13T00:46:13.427306+0000 mon.a (mon.0) 960 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:14.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:13 smithi088 bash[68960]: cluster 2024-09-13T00:46:12.543623+0000 mgr.y (mgr.25304) 302 : cluster [DBG] pgmap v571: 425 pgs: 9 active+clean+remapped, 131 active+undersized, 47 active+undersized+degraded, 238 active+clean; 2.6 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 301 KiB/s rd, 150 KiB/s wr, 625 op/s; 74/534 objects degraded (13.858%); 2/534 objects misplaced (0.375%); 0 B/s, 0 objects/s recovering 2024-09-13T00:46:14.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:13 smithi088 bash[68960]: audit 2024-09-13T00:46:12.654753+0000 mon.a (mon.0) 954 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2452864905,v1:172.21.15.77:6827/2452864905]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-13T00:46:14.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:13 smithi088 bash[68960]: cluster 2024-09-13T00:46:12.655174+0000 mon.a (mon.0) 955 : cluster [DBG] osdmap e642: 8 total, 7 up, 8 in 2024-09-13T00:46:14.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:13 smithi088 bash[68960]: audit 2024-09-13T00:46:12.658058+0000 mon.a (mon.0) 956 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2452864905,v1:172.21.15.77:6827/2452864905]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi077", "root=default"]}]: dispatch 2024-09-13T00:46:14.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:13 smithi088 bash[68960]: audit 2024-09-13T00:46:13.072521+0000 mon.a (mon.0) 957 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:14.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:13 smithi088 bash[68960]: audit 2024-09-13T00:46:13.219274+0000 mon.a (mon.0) 958 : audit [INF] from='osd.3 [v2:172.21.15.77:6826/2452864905,v1:172.21.15.77:6827/2452864905]' entity='osd.3' 2024-09-13T00:46:14.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:13 smithi088 bash[68960]: audit 2024-09-13T00:46:13.425126+0000 mon.a (mon.0) 959 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:14.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:13 smithi088 bash[68960]: audit 2024-09-13T00:46:13.427306+0000 mon.a (mon.0) 960 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:14.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:14 smithi077 bash[80681]: debug 2024-09-13T00:46:14.211+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:46:15.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:14 smithi077 bash[80681]: audit 2024-09-13T00:46:14.069483+0000 mon.a (mon.0) 961 : audit [DBG] from='client.? 172.21.15.77:0/723899379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:15.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:14 smithi077 bash[80681]: cluster 2024-09-13T00:46:14.214594+0000 mon.a (mon.0) 962 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:46:15.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:14 smithi077 bash[80681]: cluster 2024-09-13T00:46:14.214656+0000 mon.a (mon.0) 963 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T00:46:15.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:14 smithi077 bash[80681]: cluster 2024-09-13T00:46:14.231853+0000 mon.a (mon.0) 964 : cluster [INF] osd.3 [v2:172.21.15.77:6826/2452864905,v1:172.21.15.77:6827/2452864905] boot 2024-09-13T00:46:15.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:14 smithi077 bash[80681]: cluster 2024-09-13T00:46:14.231952+0000 mon.a (mon.0) 965 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2024-09-13T00:46:15.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:14 smithi077 bash[80681]: audit 2024-09-13T00:46:14.236112+0000 mon.a (mon.0) 966 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:46:15.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:14 smithi077 bash[83127]: audit 2024-09-13T00:46:14.069483+0000 mon.a (mon.0) 961 : audit [DBG] from='client.? 172.21.15.77:0/723899379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:15.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:14 smithi077 bash[83127]: cluster 2024-09-13T00:46:14.214594+0000 mon.a (mon.0) 962 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:46:15.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:14 smithi077 bash[83127]: cluster 2024-09-13T00:46:14.214656+0000 mon.a (mon.0) 963 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T00:46:15.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:14 smithi077 bash[83127]: cluster 2024-09-13T00:46:14.231853+0000 mon.a (mon.0) 964 : cluster [INF] osd.3 [v2:172.21.15.77:6826/2452864905,v1:172.21.15.77:6827/2452864905] boot 2024-09-13T00:46:15.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:14 smithi077 bash[83127]: cluster 2024-09-13T00:46:14.231952+0000 mon.a (mon.0) 965 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2024-09-13T00:46:15.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:14 smithi077 bash[83127]: audit 2024-09-13T00:46:14.236112+0000 mon.a (mon.0) 966 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:46:15.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:14 smithi088 bash[68960]: audit 2024-09-13T00:46:14.069483+0000 mon.a (mon.0) 961 : audit [DBG] from='client.? 172.21.15.77:0/723899379' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:15.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:14 smithi088 bash[68960]: cluster 2024-09-13T00:46:14.214594+0000 mon.a (mon.0) 962 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:46:15.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:14 smithi088 bash[68960]: cluster 2024-09-13T00:46:14.214656+0000 mon.a (mon.0) 963 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T00:46:15.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:14 smithi088 bash[68960]: cluster 2024-09-13T00:46:14.231853+0000 mon.a (mon.0) 964 : cluster [INF] osd.3 [v2:172.21.15.77:6826/2452864905,v1:172.21.15.77:6827/2452864905] boot 2024-09-13T00:46:15.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:14 smithi088 bash[68960]: cluster 2024-09-13T00:46:14.231952+0000 mon.a (mon.0) 965 : cluster [DBG] osdmap e643: 8 total, 8 up, 8 in 2024-09-13T00:46:15.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:14 smithi088 bash[68960]: audit 2024-09-13T00:46:14.236112+0000 mon.a (mon.0) 966 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-13T00:46:15.720 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:15.720 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:15.720 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:15.720 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:15.720 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:15.720 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.720 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:15.720 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.721 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:15.721 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:15.721 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.721 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:15.721 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.721 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:15.721 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:15.721 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.721 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:15.721 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.721 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:15.721 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:15.721 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.722 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:15.722 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.722 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:15.722 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:15.722 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.722 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:15.722 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.722 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:15.722 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:15.722 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:15.722 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:15.722 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:15.722 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:15.723 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:15.723 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:15.723 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:15.723 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:15.723 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:15.723 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:15.723 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.723 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:15.723 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.723 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:15.723 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:15.723 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.723 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:15.723 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.724 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:15.724 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:15.724 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.724 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:15.724 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.724 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:15.724 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:15.724 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.724 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:15.724 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.724 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:15.724 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:15.724 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.725 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:15.725 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.725 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:15.725 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:15.725 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:15.725 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:15.725 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:15.725 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:15.725 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:15.725 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.725 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:15.725 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:15.725 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:15.725 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.726 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:15.726 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:15.726 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:15.726 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.726 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:15.726 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:15.726 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:15.726 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.726 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:15.726 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:15.726 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:15.726 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.727 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:15.727 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:15.727 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:15.727 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 512~512 = 512 2024-09-13T00:46:15.727 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.727 INFO:tasks.workunit.client.0.smithi077.stdout:aio discard: 1536~512 = 0 2024-09-13T00:46:15.727 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:15.727 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:15.727 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:15.727 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:15.727 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:15.727 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 0 2024-09-13T00:46:15.727 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.727 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.727 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 0 2024-09-13T00:46:15.728 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.728 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.728 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 16384 2024-09-13T00:46:15.728 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.728 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.728 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 16384 2024-09-13T00:46:15.728 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.728 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.728 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:15.728 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:15.728 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 49152 2024-09-13T00:46:15.728 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.728 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.729 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 49152 2024-09-13T00:46:15.729 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.729 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.729 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 65536 2024-09-13T00:46:15.729 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.729 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.729 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 65536 2024-09-13T00:46:15.729 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.729 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.729 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:15.729 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:15.729 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 98304 2024-09-13T00:46:15.729 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.730 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.730 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 98304 2024-09-13T00:46:15.730 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.730 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.730 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 114688 2024-09-13T00:46:15.730 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.730 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.730 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 114688 2024-09-13T00:46:15.730 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.730 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.730 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:15.730 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:15.730 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 147456 2024-09-13T00:46:15.730 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.731 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.731 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 147456 2024-09-13T00:46:15.731 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.731 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.731 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 163840 2024-09-13T00:46:15.731 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.731 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.731 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 163840 2024-09-13T00:46:15.731 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.731 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.731 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:15.731 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:15.731 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 196608 2024-09-13T00:46:15.732 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.732 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.732 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 196608 2024-09-13T00:46:15.732 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.732 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.732 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 212992 2024-09-13T00:46:15.732 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.732 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.732 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 212992 2024-09-13T00:46:15.732 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.732 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.732 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:15.732 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:15.733 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.733 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:15.733 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.733 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:15.733 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:15.733 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.733 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.733 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.733 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:15.733 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.733 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:15.733 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:15.733 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.734 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.734 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.734 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:15.734 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.734 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:15.734 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:15.734 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.734 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.734 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.734 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:15.734 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.734 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:15.734 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:15.734 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.735 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.735 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.735 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:15.735 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:15.735 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.735 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:15.735 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:15.735 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.735 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:15.735 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.735 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:15.735 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:15.735 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.735 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.736 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:15.736 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:15.736 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:15.736 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:15.736 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:15.736 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:15.736 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:15.736 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:16.395 INFO:teuthology.orchestra.run.smithi077.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 4 2024-09-13T00:46:16.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:16 smithi077 bash[80681]: cluster 2024-09-13T00:46:14.546349+0000 mgr.y (mgr.25304) 303 : cluster [DBG] pgmap v574: 425 pgs: 2 remapped+peering, 63 peering, 7 active+clean+remapped, 80 active+undersized, 35 active+undersized+degraded, 238 active+clean; 6.8 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 539 KiB/s rd, 802 KiB/s wr, 648 op/s; 58/537 objects degraded (10.801%); 1/537 objects misplaced (0.186%) 2024-09-13T00:46:16.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:16 smithi077 bash[80681]: cluster 2024-09-13T00:46:15.235323+0000 mon.a (mon.0) 967 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2024-09-13T00:46:16.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:16 smithi077 bash[80681]: cluster 2024-09-13T00:46:15.963428+0000 mon.a (mon.0) 968 : cluster [WRN] Health check update: Degraded data redundancy: 58/537 objects degraded (10.801%), 35 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:16.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:16 smithi077 bash[83127]: cluster 2024-09-13T00:46:14.546349+0000 mgr.y (mgr.25304) 303 : cluster [DBG] pgmap v574: 425 pgs: 2 remapped+peering, 63 peering, 7 active+clean+remapped, 80 active+undersized, 35 active+undersized+degraded, 238 active+clean; 6.8 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 539 KiB/s rd, 802 KiB/s wr, 648 op/s; 58/537 objects degraded (10.801%); 1/537 objects misplaced (0.186%) 2024-09-13T00:46:16.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:16 smithi077 bash[83127]: cluster 2024-09-13T00:46:15.235323+0000 mon.a (mon.0) 967 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2024-09-13T00:46:16.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:16 smithi077 bash[83127]: cluster 2024-09-13T00:46:15.963428+0000 mon.a (mon.0) 968 : cluster [WRN] Health check update: Degraded data redundancy: 58/537 objects degraded (10.801%), 35 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:16.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:16 smithi088 bash[68960]: cluster 2024-09-13T00:46:14.546349+0000 mgr.y (mgr.25304) 303 : cluster [DBG] pgmap v574: 425 pgs: 2 remapped+peering, 63 peering, 7 active+clean+remapped, 80 active+undersized, 35 active+undersized+degraded, 238 active+clean; 6.8 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 539 KiB/s rd, 802 KiB/s wr, 648 op/s; 58/537 objects degraded (10.801%); 1/537 objects misplaced (0.186%) 2024-09-13T00:46:16.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:16 smithi088 bash[68960]: cluster 2024-09-13T00:46:15.235323+0000 mon.a (mon.0) 967 : cluster [DBG] osdmap e644: 8 total, 8 up, 8 in 2024-09-13T00:46:16.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:16 smithi088 bash[68960]: cluster 2024-09-13T00:46:15.963428+0000 mon.a (mon.0) 968 : cluster [WRN] Health check update: Degraded data redundancy: 58/537 objects degraded (10.801%), 35 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:17.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:17 smithi077 bash[83127]: cluster 2024-09-13T00:46:16.242849+0000 mon.a (mon.0) 969 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2024-09-13T00:46:17.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:17 smithi077 bash[83127]: audit 2024-09-13T00:46:16.385806+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.77:0/484706500' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:17.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:17 smithi077 bash[80681]: cluster 2024-09-13T00:46:16.242849+0000 mon.a (mon.0) 969 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2024-09-13T00:46:17.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:17 smithi077 bash[80681]: audit 2024-09-13T00:46:16.385806+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.77:0/484706500' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:17.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:17 smithi088 bash[68960]: cluster 2024-09-13T00:46:16.242849+0000 mon.a (mon.0) 969 : cluster [DBG] osdmap e645: 8 total, 8 up, 8 in 2024-09-13T00:46:17.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:17 smithi088 bash[68960]: audit 2024-09-13T00:46:16.385806+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.77:0/484706500' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-09-13T00:46:18.161 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph orch upgrade pause' 2024-09-13T00:46:18.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:18 smithi077 bash[80681]: cluster 2024-09-13T00:46:16.547109+0000 mgr.y (mgr.25304) 304 : cluster [DBG] pgmap v577: 425 pgs: 2 remapped+peering, 78 peering, 7 active+clean+remapped, 54 active+undersized, 29 active+undersized+degraded, 255 active+clean; 6.8 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 922 KiB/s rd, 1.5 MiB/s wr, 927 op/s; 44/537 objects degraded (8.194%); 1/537 objects misplaced (0.186%); 8 B/s, 0 objects/s recovering 2024-09-13T00:46:18.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:18 smithi077 bash[83127]: cluster 2024-09-13T00:46:16.547109+0000 mgr.y (mgr.25304) 304 : cluster [DBG] pgmap v577: 425 pgs: 2 remapped+peering, 78 peering, 7 active+clean+remapped, 54 active+undersized, 29 active+undersized+degraded, 255 active+clean; 6.8 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 922 KiB/s rd, 1.5 MiB/s wr, 927 op/s; 44/537 objects degraded (8.194%); 1/537 objects misplaced (0.186%); 8 B/s, 0 objects/s recovering 2024-09-13T00:46:18.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:18 smithi088 bash[68960]: cluster 2024-09-13T00:46:16.547109+0000 mgr.y (mgr.25304) 304 : cluster [DBG] pgmap v577: 425 pgs: 2 remapped+peering, 78 peering, 7 active+clean+remapped, 54 active+undersized, 29 active+undersized+degraded, 255 active+clean; 6.8 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 922 KiB/s rd, 1.5 MiB/s wr, 927 op/s; 44/537 objects degraded (8.194%); 1/537 objects misplaced (0.186%); 8 B/s, 0 objects/s recovering 2024-09-13T00:46:18.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:46:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:46:18] "GET /metrics HTTP/1.1" 200 60561 "" "Prometheus/2.43.0" 2024-09-13T00:46:20.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:20 smithi077 bash[80681]: cluster 2024-09-13T00:46:18.548668+0000 mgr.y (mgr.25304) 305 : cluster [DBG] pgmap v578: 425 pgs: 1 remapped+peering, 78 peering, 4 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 321 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 626 KiB/s rd, 1.0 MiB/s wr, 629 op/s; 8/537 objects degraded (1.490%); 1/537 objects misplaced (0.186%); 6 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T00:46:20.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:20 smithi077 bash[83127]: cluster 2024-09-13T00:46:18.548668+0000 mgr.y (mgr.25304) 305 : cluster [DBG] pgmap v578: 425 pgs: 1 remapped+peering, 78 peering, 4 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 321 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 626 KiB/s rd, 1.0 MiB/s wr, 629 op/s; 8/537 objects degraded (1.490%); 1/537 objects misplaced (0.186%); 6 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T00:46:20.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:20 smithi088 bash[68960]: cluster 2024-09-13T00:46:18.548668+0000 mgr.y (mgr.25304) 305 : cluster [DBG] pgmap v578: 425 pgs: 1 remapped+peering, 78 peering, 4 active+clean+remapped, 15 active+undersized, 6 active+undersized+degraded, 321 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 626 KiB/s rd, 1.0 MiB/s wr, 629 op/s; 8/537 objects degraded (1.490%); 1/537 objects misplaced (0.186%); 6 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T00:46:21.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:21 smithi077 bash[80681]: cluster 2024-09-13T00:46:20.969438+0000 mon.a (mon.0) 970 : cluster [WRN] Health check update: Degraded data redundancy: 8/537 objects degraded (1.490%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:21.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:21 smithi077 bash[83127]: cluster 2024-09-13T00:46:20.969438+0000 mon.a (mon.0) 970 : cluster [WRN] Health check update: Degraded data redundancy: 8/537 objects degraded (1.490%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:21.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:21 smithi088 bash[68960]: cluster 2024-09-13T00:46:20.969438+0000 mon.a (mon.0) 970 : cluster [WRN] Health check update: Degraded data redundancy: 8/537 objects degraded (1.490%), 6 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:22.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:22 smithi077 bash[80681]: cluster 2024-09-13T00:46:20.551892+0000 mgr.y (mgr.25304) 306 : cluster [DBG] pgmap v579: 425 pgs: 425 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 589 KiB/s wr, 1.12k op/s; 33 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T00:46:22.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:22 smithi077 bash[80681]: cluster 2024-09-13T00:46:21.263009+0000 mon.a (mon.0) 971 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/537 objects degraded (1.490%), 6 pgs degraded) 2024-09-13T00:46:22.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:22 smithi077 bash[83127]: cluster 2024-09-13T00:46:20.551892+0000 mgr.y (mgr.25304) 306 : cluster [DBG] pgmap v579: 425 pgs: 425 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 589 KiB/s wr, 1.12k op/s; 33 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T00:46:22.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:22 smithi077 bash[83127]: cluster 2024-09-13T00:46:21.263009+0000 mon.a (mon.0) 971 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/537 objects degraded (1.490%), 6 pgs degraded) 2024-09-13T00:46:22.600 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:46:22 smithi088 bash[61791]: ts=2024-09-13T00:46:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:46:22.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:22 smithi088 bash[68960]: cluster 2024-09-13T00:46:20.551892+0000 mgr.y (mgr.25304) 306 : cluster [DBG] pgmap v579: 425 pgs: 425 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 589 KiB/s wr, 1.12k op/s; 33 KiB/s, 1 keys/s, 0 objects/s recovering 2024-09-13T00:46:22.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:22 smithi088 bash[68960]: cluster 2024-09-13T00:46:21.263009+0000 mon.a (mon.0) 971 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 8/537 objects degraded (1.490%), 6 pgs degraded) 2024-09-13T00:46:22.716 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.716 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:22.717 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:22.717 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:22.717 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:22.717 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:22.717 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.717 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.717 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:22.717 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:22.717 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:22.717 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:22.717 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:22.717 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.717 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.718 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:22.718 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.718 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.718 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:22.718 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.718 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.718 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:22.718 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:22.718 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:22.718 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:22.718 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:22.718 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.718 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.719 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:22.719 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:22.719 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:22.719 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:22.719 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:22.719 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.719 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.719 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:22.719 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:22.719 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:22.719 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:22.719 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:22.719 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.720 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.720 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:22.720 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:22.720 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:22.720 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:22.720 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:22.720 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.720 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.720 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:22.720 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.720 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.720 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:22.720 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.720 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.721 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:22.721 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:22.721 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:22.721 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:22.721 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:22.721 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.721 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.721 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:22.721 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:22.721 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:22.721 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:22.721 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:22.721 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.722 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.722 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:22.722 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:22.722 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:22.722 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:22.722 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:22.722 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.722 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.722 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:22.722 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:22.722 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:22.722 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:22.722 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:22.722 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.722 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.723 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:22.723 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.723 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.723 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:22.723 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.723 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.723 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:22.723 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:22.723 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:22.723 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:22.723 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:22.723 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.723 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.724 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:22.724 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:22.724 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:22.724 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:22.724 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:22.724 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.724 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.724 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:22.724 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:22.724 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:22.724 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:22.724 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:22.724 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.724 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.725 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:22.725 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:22.725 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:22.725 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:22.725 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:22.725 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.725 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.725 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:22.725 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:22.725 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:22.725 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:22.725 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:22.725 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:22.725 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:22.726 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:22.726 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 4 2024-09-13T00:46:22.726 INFO:tasks.workunit.client.0.smithi077.stdout:read: 4 2024-09-13T00:46:22.726 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS1 (16999 ms) 2024-09-13T00:46:22.726 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS2 2024-09-13T00:46:22.726 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:23.167 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:46:24.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[80681]: cluster 2024-09-13T00:46:22.553201+0000 mgr.y (mgr.25304) 307 : cluster [DBG] pgmap v580: 425 pgs: 425 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 421 KiB/s rd, 209 KiB/s wr, 423 op/s; 26 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T00:46:24.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[80681]: audit 2024-09-13T00:46:22.696384+0000 mon.a (mon.0) 972 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:24.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[80681]: audit 2024-09-13T00:46:22.704344+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:24.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[80681]: audit 2024-09-13T00:46:22.705760+0000 mon.a (mon.0) 974 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:46:24.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[80681]: audit 2024-09-13T00:46:22.706890+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:46:24.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[80681]: audit 2024-09-13T00:46:22.714922+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:24.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[80681]: audit 2024-09-13T00:46:22.836487+0000 mon.a (mon.0) 977 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:46:24.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[80681]: audit 2024-09-13T00:46:22.840315+0000 mon.a (mon.0) 978 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:46:24.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[80681]: audit 2024-09-13T00:46:22.842562+0000 mon.a (mon.0) 979 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:46:24.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[80681]: audit 2024-09-13T00:46:22.844244+0000 mon.a (mon.0) 980 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:46:24.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[80681]: audit 2024-09-13T00:46:22.846243+0000 mon.a (mon.0) 981 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T00:46:24.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[80681]: audit 2024-09-13T00:46:22.846516+0000 mgr.y (mgr.25304) 308 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T00:46:24.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[80681]: cephadm 2024-09-13T00:46:22.848294+0000 mgr.y (mgr.25304) 309 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-09-13T00:46:24.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[83127]: cluster 2024-09-13T00:46:22.553201+0000 mgr.y (mgr.25304) 307 : cluster [DBG] pgmap v580: 425 pgs: 425 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 421 KiB/s rd, 209 KiB/s wr, 423 op/s; 26 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T00:46:24.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[83127]: audit 2024-09-13T00:46:22.696384+0000 mon.a (mon.0) 972 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:24.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[83127]: audit 2024-09-13T00:46:22.704344+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:24.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[83127]: audit 2024-09-13T00:46:22.705760+0000 mon.a (mon.0) 974 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:46:24.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[83127]: audit 2024-09-13T00:46:22.706890+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:46:24.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[83127]: audit 2024-09-13T00:46:22.714922+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:24.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[83127]: audit 2024-09-13T00:46:22.836487+0000 mon.a (mon.0) 977 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:46:24.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[83127]: audit 2024-09-13T00:46:22.840315+0000 mon.a (mon.0) 978 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:46:24.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[83127]: audit 2024-09-13T00:46:22.842562+0000 mon.a (mon.0) 979 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:46:24.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[83127]: audit 2024-09-13T00:46:22.844244+0000 mon.a (mon.0) 980 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:46:24.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[83127]: audit 2024-09-13T00:46:22.846243+0000 mon.a (mon.0) 981 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T00:46:24.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[83127]: audit 2024-09-13T00:46:22.846516+0000 mgr.y (mgr.25304) 308 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T00:46:24.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:23 smithi077 bash[83127]: cephadm 2024-09-13T00:46:22.848294+0000 mgr.y (mgr.25304) 309 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-09-13T00:46:24.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:23 smithi088 bash[68960]: cluster 2024-09-13T00:46:22.553201+0000 mgr.y (mgr.25304) 307 : cluster [DBG] pgmap v580: 425 pgs: 425 active+clean; 6.8 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 421 KiB/s rd, 209 KiB/s wr, 423 op/s; 26 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T00:46:24.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:23 smithi088 bash[68960]: audit 2024-09-13T00:46:22.696384+0000 mon.a (mon.0) 972 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:24.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:23 smithi088 bash[68960]: audit 2024-09-13T00:46:22.704344+0000 mon.a (mon.0) 973 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:24.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:23 smithi088 bash[68960]: audit 2024-09-13T00:46:22.705760+0000 mon.a (mon.0) 974 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:46:24.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:23 smithi088 bash[68960]: audit 2024-09-13T00:46:22.706890+0000 mon.a (mon.0) 975 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:46:24.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:23 smithi088 bash[68960]: audit 2024-09-13T00:46:22.714922+0000 mon.a (mon.0) 976 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:24.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:23 smithi088 bash[68960]: audit 2024-09-13T00:46:22.836487+0000 mon.a (mon.0) 977 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:46:24.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:23 smithi088 bash[68960]: audit 2024-09-13T00:46:22.840315+0000 mon.a (mon.0) 978 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:46:24.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:23 smithi088 bash[68960]: audit 2024-09-13T00:46:22.842562+0000 mon.a (mon.0) 979 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:46:24.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:23 smithi088 bash[68960]: audit 2024-09-13T00:46:22.844244+0000 mon.a (mon.0) 980 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:46:24.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:23 smithi088 bash[68960]: audit 2024-09-13T00:46:22.846243+0000 mon.a (mon.0) 981 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T00:46:24.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:23 smithi088 bash[68960]: audit 2024-09-13T00:46:22.846516+0000 mgr.y (mgr.25304) 308 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-09-13T00:46:24.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:23 smithi088 bash[68960]: cephadm 2024-09-13T00:46:22.848294+0000 mgr.y (mgr.25304) 309 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-09-13T00:46:25.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:25 smithi088 bash[68960]: cluster 2024-09-13T00:46:24.555441+0000 mgr.y (mgr.25304) 310 : cluster [DBG] pgmap v581: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 495 KiB/s rd, 225 KiB/s wr, 468 op/s; 22 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T00:46:26.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:25 smithi077 bash[80681]: cluster 2024-09-13T00:46:24.555441+0000 mgr.y (mgr.25304) 310 : cluster [DBG] pgmap v581: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 495 KiB/s rd, 225 KiB/s wr, 468 op/s; 22 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T00:46:26.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:25 smithi077 bash[83127]: cluster 2024-09-13T00:46:24.555441+0000 mgr.y (mgr.25304) 310 : cluster [DBG] pgmap v581: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 495 KiB/s rd, 225 KiB/s wr, 468 op/s; 22 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T00:46:27.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:26 smithi077 bash[80681]: cephadm 2024-09-13T00:46:25.763646+0000 mgr.y (mgr.25304) 311 : cephadm [INF] Upgrade: Updating osd.4 2024-09-13T00:46:27.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:26 smithi077 bash[80681]: audit 2024-09-13T00:46:25.770452+0000 mon.a (mon.0) 982 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:27.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:26 smithi077 bash[80681]: audit 2024-09-13T00:46:25.771603+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T00:46:27.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:26 smithi077 bash[80681]: audit 2024-09-13T00:46:25.772459+0000 mon.a (mon.0) 984 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:46:27.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:26 smithi077 bash[80681]: cephadm 2024-09-13T00:46:25.777871+0000 mgr.y (mgr.25304) 312 : cephadm [INF] Deploying daemon osd.4 on smithi088 2024-09-13T00:46:27.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:26 smithi077 bash[83127]: cephadm 2024-09-13T00:46:25.763646+0000 mgr.y (mgr.25304) 311 : cephadm [INF] Upgrade: Updating osd.4 2024-09-13T00:46:27.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:26 smithi077 bash[83127]: audit 2024-09-13T00:46:25.770452+0000 mon.a (mon.0) 982 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:27.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:26 smithi077 bash[83127]: audit 2024-09-13T00:46:25.771603+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T00:46:27.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:26 smithi077 bash[83127]: audit 2024-09-13T00:46:25.772459+0000 mon.a (mon.0) 984 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:46:27.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:26 smithi077 bash[83127]: cephadm 2024-09-13T00:46:25.777871+0000 mgr.y (mgr.25304) 312 : cephadm [INF] Deploying daemon osd.4 on smithi088 2024-09-13T00:46:27.057 INFO:teuthology.orchestra.run.smithi077.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:46:27.073 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:26 smithi088 bash[68960]: cephadm 2024-09-13T00:46:25.763646+0000 mgr.y (mgr.25304) 311 : cephadm [INF] Upgrade: Updating osd.4 2024-09-13T00:46:27.074 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:26 smithi088 bash[68960]: audit 2024-09-13T00:46:25.770452+0000 mon.a (mon.0) 982 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:27.074 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:26 smithi088 bash[68960]: audit 2024-09-13T00:46:25.771603+0000 mon.a (mon.0) 983 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-13T00:46:27.074 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:26 smithi088 bash[68960]: audit 2024-09-13T00:46:25.772459+0000 mon.a (mon.0) 984 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:46:27.074 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:26 smithi088 bash[68960]: cephadm 2024-09-13T00:46:25.777871+0000 mgr.y (mgr.25304) 312 : cephadm [INF] Deploying daemon osd.4 on smithi088 2024-09-13T00:46:27.901 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph orch ps' 2024-09-13T00:46:28.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:28 smithi077 bash[80681]: cluster 2024-09-13T00:46:26.556503+0000 mgr.y (mgr.25304) 313 : cluster [DBG] pgmap v582: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 447 KiB/s rd, 203 KiB/s wr, 423 op/s; 20 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T00:46:28.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:28 smithi077 bash[80681]: audit 2024-09-13T00:46:27.056385+0000 mgr.y (mgr.25304) 314 : audit [DBG] from='client.44468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:46:28.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:28 smithi077 bash[80681]: cephadm 2024-09-13T00:46:27.056784+0000 mgr.y (mgr.25304) 315 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:46:28.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:28 smithi077 bash[80681]: audit 2024-09-13T00:46:27.061060+0000 mon.a (mon.0) 985 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:28.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:28 smithi077 bash[83127]: cluster 2024-09-13T00:46:26.556503+0000 mgr.y (mgr.25304) 313 : cluster [DBG] pgmap v582: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 447 KiB/s rd, 203 KiB/s wr, 423 op/s; 20 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T00:46:28.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:28 smithi077 bash[83127]: audit 2024-09-13T00:46:27.056385+0000 mgr.y (mgr.25304) 314 : audit [DBG] from='client.44468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:46:28.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:28 smithi077 bash[83127]: cephadm 2024-09-13T00:46:27.056784+0000 mgr.y (mgr.25304) 315 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:46:28.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:28 smithi077 bash[83127]: audit 2024-09-13T00:46:27.061060+0000 mon.a (mon.0) 985 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:28.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:28 smithi088 bash[68960]: cluster 2024-09-13T00:46:26.556503+0000 mgr.y (mgr.25304) 313 : cluster [DBG] pgmap v582: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 447 KiB/s rd, 203 KiB/s wr, 423 op/s; 20 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T00:46:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:28 smithi088 bash[68960]: audit 2024-09-13T00:46:27.056385+0000 mgr.y (mgr.25304) 314 : audit [DBG] from='client.44468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:46:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:28 smithi088 bash[68960]: cephadm 2024-09-13T00:46:27.056784+0000 mgr.y (mgr.25304) 315 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:ef14fa6264e59c23a50911ff257310d925626ef2ed8aade654f62752810262e6 2024-09-13T00:46:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:28 smithi088 bash[68960]: audit 2024-09-13T00:46:27.061060+0000 mon.a (mon.0) 985 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:28.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:46:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:46:28] "GET /metrics HTTP/1.1" 200 60697 "" "Prometheus/2.43.0" 2024-09-13T00:46:29.348 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:29 smithi088 bash[68960]: audit 2024-09-13T00:46:28.099993+0000 mon.a (mon.0) 986 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:29.348 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:29 smithi088 bash[68960]: audit 2024-09-13T00:46:28.424414+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:29.348 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:29 smithi088 bash[68960]: audit 2024-09-13T00:46:28.426770+0000 mon.a (mon.0) 988 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:29.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:29 smithi077 bash[83127]: audit 2024-09-13T00:46:28.099993+0000 mon.a (mon.0) 986 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:29.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:29 smithi077 bash[83127]: audit 2024-09-13T00:46:28.424414+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:29.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:29 smithi077 bash[83127]: audit 2024-09-13T00:46:28.426770+0000 mon.a (mon.0) 988 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:29.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:29 smithi077 bash[80681]: audit 2024-09-13T00:46:28.099993+0000 mon.a (mon.0) 986 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:29.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:29 smithi077 bash[80681]: audit 2024-09-13T00:46:28.424414+0000 mon.a (mon.0) 987 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:29.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:29 smithi077 bash[80681]: audit 2024-09-13T00:46:28.426770+0000 mon.a (mon.0) 988 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:30.519 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:46:30 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:30.519 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 00:46:30 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:30.519 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:46:30 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:30.520 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:46:30 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:30.520 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:46:30 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:30.520 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:46:30 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:30.521 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:30 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:30.521 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:30 smithi088 bash[68960]: cluster 2024-09-13T00:46:28.558010+0000 mgr.y (mgr.25304) 316 : cluster [DBG] pgmap v583: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 385 KiB/s rd, 175 KiB/s wr, 364 op/s; 17 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T00:46:30.521 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:30 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:30.521 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:46:30 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:30.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:30 smithi077 bash[80681]: cluster 2024-09-13T00:46:28.558010+0000 mgr.y (mgr.25304) 316 : cluster [DBG] pgmap v583: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 385 KiB/s rd, 175 KiB/s wr, 364 op/s; 17 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T00:46:30.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:30 smithi077 bash[83127]: cluster 2024-09-13T00:46:28.558010+0000 mgr.y (mgr.25304) 316 : cluster [DBG] pgmap v583: 425 pgs: 425 active+clean; 11 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 385 KiB/s rd, 175 KiB/s wr, 364 op/s; 17 KiB/s, 0 keys/s, 0 objects/s recovering 2024-09-13T00:46:30.851 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:30 smithi088 systemd[1]: Stopping Ceph osd.4 for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:46:30.851 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:30 smithi088 bash[28391]: debug 2024-09-13T00:46:30.620+0000 7fd2fb83c640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:46:30.851 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:30 smithi088 bash[28391]: debug 2024-09-13T00:46:30.620+0000 7fd2fb83c640 -1 osd.4 645 *** Got signal Terminated *** 2024-09-13T00:46:30.851 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:30 smithi088 bash[28391]: debug 2024-09-13T00:46:30.620+0000 7fd2fb83c640 -1 osd.4 645 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T00:46:31.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:31 smithi077 bash[80681]: debug 2024-09-13T00:46:31.423+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:46:31.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:31 smithi077 bash[80681]: cluster 2024-09-13T00:46:30.622941+0000 mon.a (mon.0) 989 : cluster [INF] osd.4 marked itself down and dead 2024-09-13T00:46:31.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:31 smithi077 bash[83127]: cluster 2024-09-13T00:46:30.622941+0000 mon.a (mon.0) 989 : cluster [INF] osd.4 marked itself down and dead 2024-09-13T00:46:31.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:31 smithi088 bash[68960]: cluster 2024-09-13T00:46:30.622941+0000 mon.a (mon.0) 989 : cluster [INF] osd.4 marked itself down and dead 2024-09-13T00:46:32.600 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:46:32 smithi088 bash[61791]: ts=2024-09-13T00:46:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:46:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:32 smithi088 bash[68960]: cluster 2024-09-13T00:46:30.561306+0000 mgr.y (mgr.25304) 317 : cluster [DBG] pgmap v584: 425 pgs: 425 active+clean; 19 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 667 KiB/s rd, 896 KiB/s wr, 438 op/s; 17 KiB/s, 0 objects/s recovering 2024-09-13T00:46:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:32 smithi088 bash[68960]: cluster 2024-09-13T00:46:31.427871+0000 mon.a (mon.0) 990 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T00:46:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:32 smithi088 bash[68960]: cluster 2024-09-13T00:46:31.427918+0000 mon.a (mon.0) 991 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:46:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:32 smithi088 bash[68960]: cluster 2024-09-13T00:46:31.445303+0000 mon.a (mon.0) 992 : cluster [DBG] osdmap e646: 8 total, 7 up, 8 in 2024-09-13T00:46:32.691 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:46:32.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:32 smithi077 bash[80681]: cluster 2024-09-13T00:46:30.561306+0000 mgr.y (mgr.25304) 317 : cluster [DBG] pgmap v584: 425 pgs: 425 active+clean; 19 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 667 KiB/s rd, 896 KiB/s wr, 438 op/s; 17 KiB/s, 0 objects/s recovering 2024-09-13T00:46:32.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:32 smithi077 bash[80681]: cluster 2024-09-13T00:46:31.427871+0000 mon.a (mon.0) 990 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T00:46:32.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:32 smithi077 bash[80681]: cluster 2024-09-13T00:46:31.427918+0000 mon.a (mon.0) 991 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:46:32.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:32 smithi077 bash[80681]: cluster 2024-09-13T00:46:31.445303+0000 mon.a (mon.0) 992 : cluster [DBG] osdmap e646: 8 total, 7 up, 8 in 2024-09-13T00:46:32.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:32 smithi077 bash[83127]: cluster 2024-09-13T00:46:30.561306+0000 mgr.y (mgr.25304) 317 : cluster [DBG] pgmap v584: 425 pgs: 425 active+clean; 19 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 667 KiB/s rd, 896 KiB/s wr, 438 op/s; 17 KiB/s, 0 objects/s recovering 2024-09-13T00:46:32.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:32 smithi077 bash[83127]: cluster 2024-09-13T00:46:31.427871+0000 mon.a (mon.0) 990 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-09-13T00:46:32.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:32 smithi077 bash[83127]: cluster 2024-09-13T00:46:31.427918+0000 mon.a (mon.0) 991 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:46:32.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:32 smithi077 bash[83127]: cluster 2024-09-13T00:46:31.445303+0000 mon.a (mon.0) 992 : cluster [DBG] osdmap e646: 8 total, 7 up, 8 in 2024-09-13T00:46:33.100 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:32 smithi088 bash[74032]: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-4 2024-09-13T00:46:33.100 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:33 smithi088 bash[74099]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-4 2024-09-13T00:46:33.666 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:33.666 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:33.666 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:33.666 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:33.666 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:33.666 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.666 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:33.666 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.666 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:33.666 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:33.667 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.667 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:33.667 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.667 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:33.667 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:33.667 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.667 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:33.667 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.667 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:33.667 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:33.667 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.667 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:33.667 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.668 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:33.668 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:33.668 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.668 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:33.668 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.668 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:33.668 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:33.668 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:33.668 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:33.668 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:33.668 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:33.668 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:33.669 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:33.669 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:33.669 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:33.669 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:33.669 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:33.669 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.669 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:33.669 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.669 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:33.669 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:33.669 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.669 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:33.669 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.669 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:33.670 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:33.670 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.670 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:33.670 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.670 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:33.670 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:33.670 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.670 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:33.670 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.670 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:33.670 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:33.670 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.670 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:33.670 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.671 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:33.671 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:33.671 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:33.671 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:33.671 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:33.671 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:33.671 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:33.671 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.671 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:33.671 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:33.671 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:33.671 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.671 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:33.671 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:33.672 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:33.672 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.672 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:33.672 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:33.672 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:33.672 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.672 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:33.672 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:33.672 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:33.672 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.672 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:33.672 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:33.672 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:33.672 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 512~512 = 512 2024-09-13T00:46:33.673 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.673 INFO:tasks.workunit.client.0.smithi077.stdout:aio discard: 1536~512 = 0 2024-09-13T00:46:33.673 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:33.673 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:33.673 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:33.673 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:33.673 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:33.673 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 0 2024-09-13T00:46:33.673 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.673 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.673 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 0 2024-09-13T00:46:33.673 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.673 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.673 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 16384 2024-09-13T00:46:33.674 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.674 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.674 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 16384 2024-09-13T00:46:33.674 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.674 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.674 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:33.674 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:33.674 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 49152 2024-09-13T00:46:33.674 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.674 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.674 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 49152 2024-09-13T00:46:33.674 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.674 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.675 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 65536 2024-09-13T00:46:33.675 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.675 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.675 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 65536 2024-09-13T00:46:33.675 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.675 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.675 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:33.675 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:33.675 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 98304 2024-09-13T00:46:33.675 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.675 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.675 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 98304 2024-09-13T00:46:33.675 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.676 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.676 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 114688 2024-09-13T00:46:33.676 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.676 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.676 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 114688 2024-09-13T00:46:33.676 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.676 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.676 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:33.676 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:33.676 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 147456 2024-09-13T00:46:33.676 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.676 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.676 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 147456 2024-09-13T00:46:33.676 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.677 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.677 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 163840 2024-09-13T00:46:33.677 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.677 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.677 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 163840 2024-09-13T00:46:33.677 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.677 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.678 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:33.678 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:33.678 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 196608 2024-09-13T00:46:33.678 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.678 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.678 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 196608 2024-09-13T00:46:33.678 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.678 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.678 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 212992 2024-09-13T00:46:33.678 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.678 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.679 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 212992 2024-09-13T00:46:33.679 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.679 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.679 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:33.679 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:33.679 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.679 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:33.679 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.679 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:33.679 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:33.679 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.679 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.679 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.680 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:33.680 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.680 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:33.680 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:33.680 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.680 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.680 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.680 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:33.680 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.680 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:33.680 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:33.680 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.680 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.681 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.681 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:33.681 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.681 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:33.681 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:33.681 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.681 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.681 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.681 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:33.681 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:33.681 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.681 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:33.681 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:33.681 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.682 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:33.682 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.682 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:33.682 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:33.682 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.682 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.682 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.682 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:33.682 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:33.682 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:33.682 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:33.682 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:33.682 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:33.683 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:33.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:33 smithi077 bash[80681]: cluster 2024-09-13T00:46:32.448798+0000 mon.a (mon.0) 993 : cluster [DBG] osdmap e647: 8 total, 7 up, 8 in 2024-09-13T00:46:33.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:33 smithi077 bash[83127]: cluster 2024-09-13T00:46:32.448798+0000 mon.a (mon.0) 993 : cluster [DBG] osdmap e647: 8 total, 7 up, 8 in 2024-09-13T00:46:33.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:33 smithi088 bash[68960]: cluster 2024-09-13T00:46:32.448798+0000 mon.a (mon.0) 993 : cluster [DBG] osdmap e647: 8 total, 7 up, 8 in 2024-09-13T00:46:34.532 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:34 smithi088 bash[68960]: cluster 2024-09-13T00:46:32.561908+0000 mgr.y (mgr.25304) 318 : cluster [DBG] pgmap v587: 425 pgs: 53 stale+active+clean, 372 active+clean; 19 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 423 KiB/s rd, 1.1 MiB/s wr, 111 op/s 2024-09-13T00:46:34.723 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:34 smithi077 bash[80681]: cluster 2024-09-13T00:46:32.561908+0000 mgr.y (mgr.25304) 318 : cluster [DBG] pgmap v587: 425 pgs: 53 stale+active+clean, 372 active+clean; 19 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 423 KiB/s rd, 1.1 MiB/s wr, 111 op/s 2024-09-13T00:46:34.723 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:34 smithi077 bash[83127]: cluster 2024-09-13T00:46:32.561908+0000 mgr.y (mgr.25304) 318 : cluster [DBG] pgmap v587: 425 pgs: 53 stale+active+clean, 372 active+clean; 19 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 423 KiB/s rd, 1.1 MiB/s wr, 111 op/s 2024-09-13T00:46:35.478 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:35.478 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:46:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:35.479 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 00:46:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:35.479 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 00:46:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:35.479 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 00:46:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:35.480 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:46:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:35.480 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 00:46:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:35.480 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:46:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:35.480 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:35 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.4.service: Deactivated successfully. 2024-09-13T00:46:35.481 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:35 smithi088 systemd[1]: Stopped Ceph osd.4 for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:46:35.481 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:35 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T00:46:35.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:35 smithi077 bash[80681]: cluster 2024-09-13T00:46:34.563747+0000 mgr.y (mgr.25304) 319 : cluster [DBG] pgmap v588: 425 pgs: 73 active+undersized, 16 stale+active+clean, 40 active+undersized+degraded, 296 active+clean; 27 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 662 op/s; 60/564 objects degraded (10.638%) 2024-09-13T00:46:35.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:35 smithi077 bash[80681]: cluster 2024-09-13T00:46:35.450881+0000 mon.a (mon.0) 994 : cluster [WRN] Health check failed: Degraded data redundancy: 60/564 objects degraded (10.638%), 40 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:35.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:35 smithi077 bash[83127]: cluster 2024-09-13T00:46:34.563747+0000 mgr.y (mgr.25304) 319 : cluster [DBG] pgmap v588: 425 pgs: 73 active+undersized, 16 stale+active+clean, 40 active+undersized+degraded, 296 active+clean; 27 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 662 op/s; 60/564 objects degraded (10.638%) 2024-09-13T00:46:35.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:35 smithi077 bash[83127]: cluster 2024-09-13T00:46:35.450881+0000 mon.a (mon.0) 994 : cluster [WRN] Health check failed: Degraded data redundancy: 60/564 objects degraded (10.638%), 40 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:35.850 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:35 smithi088 systemd[1]: Started Ceph osd.4 for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:46:35.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:35 smithi088 bash[68960]: cluster 2024-09-13T00:46:34.563747+0000 mgr.y (mgr.25304) 319 : cluster [DBG] pgmap v588: 425 pgs: 73 active+undersized, 16 stale+active+clean, 40 active+undersized+degraded, 296 active+clean; 27 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 662 op/s; 60/564 objects degraded (10.638%) 2024-09-13T00:46:35.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:35 smithi088 bash[68960]: cluster 2024-09-13T00:46:35.450881+0000 mon.a (mon.0) 994 : cluster [WRN] Health check failed: Degraded data redundancy: 60/564 objects degraded (10.638%), 40 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:36.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:36 smithi077 bash[80681]: audit 2024-09-13T00:46:35.518953+0000 mon.a (mon.0) 995 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:36.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:36 smithi077 bash[80681]: audit 2024-09-13T00:46:35.526477+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:36.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:36 smithi077 bash[80681]: audit 2024-09-13T00:46:35.527663+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:46:36.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:36 smithi077 bash[83127]: audit 2024-09-13T00:46:35.518953+0000 mon.a (mon.0) 995 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:36.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:36 smithi077 bash[83127]: audit 2024-09-13T00:46:35.526477+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:36.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:36 smithi077 bash[83127]: audit 2024-09-13T00:46:35.527663+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:46:36.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:36 smithi088 bash[68960]: audit 2024-09-13T00:46:35.518953+0000 mon.a (mon.0) 995 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:36.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:36 smithi088 bash[68960]: audit 2024-09-13T00:46:35.526477+0000 mon.a (mon.0) 996 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:36.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:36 smithi088 bash[68960]: audit 2024-09-13T00:46:35.527663+0000 mon.a (mon.0) 997 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:46:37.176 INFO:teuthology.orchestra.run.smithi077.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T00:46:37.177 INFO:teuthology.orchestra.run.smithi077.stdout:alertmanager.a smithi077 *:9093,9094 running (9m) 26s ago 16m 14.8M - 0.25.0 c8568f914cd2 49a4b5607300 2024-09-13T00:46:37.177 INFO:teuthology.orchestra.run.smithi077.stdout:grafana.a smithi088 *:3000 running (9m) 59s ago 16m 73.1M - 9.4.7 954c08fa6188 c369fdf58d31 2024-09-13T00:46:37.177 INFO:teuthology.orchestra.run.smithi077.stdout:mds.foo.smithi077.bigbbf smithi077 running (14m) 26s ago 14m 14.7M - 17.2.7-1627-gbbdadfa3 214985049bdb 30b25ab8f2dd 2024-09-13T00:46:37.177 INFO:teuthology.orchestra.run.smithi077.stdout:mds.foo.smithi088.krmowh smithi088 running (14m) 59s ago 14m 19.5M - 17.2.7-1627-gbbdadfa3 214985049bdb ebef6ef73335 2024-09-13T00:46:37.177 INFO:teuthology.orchestra.run.smithi077.stdout:mgr.x smithi088 *:8443,9283,8765 running (8m) 59s ago 25m 427M - 18.2.4-837-g7a853b17 f21ba1eea74e fdfc79d860d1 2024-09-13T00:46:37.177 INFO:teuthology.orchestra.run.smithi077.stdout:mgr.y smithi077 *:8443,9283,8765 running (8m) 26s ago 27m 497M - 18.2.4-837-g7a853b17 f21ba1eea74e d46e77b45675 2024-09-13T00:46:37.177 INFO:teuthology.orchestra.run.smithi077.stdout:mon.a smithi077 running (7m) 26s ago 27m 74.1M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 3edacf970db4 2024-09-13T00:46:37.177 INFO:teuthology.orchestra.run.smithi077.stdout:mon.b smithi088 running (5m) 59s ago 25m 44.6M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e dd7e71fd2ce3 2024-09-13T00:46:37.177 INFO:teuthology.orchestra.run.smithi077.stdout:mon.c smithi077 running (5m) 26s ago 25m 59.8M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 0a377e5bd304 2024-09-13T00:46:37.177 INFO:teuthology.orchestra.run.smithi077.stdout:node-exporter.a smithi077 *:9100 running (9m) 26s ago 17m 8576k - 1.5.0 0da6a335fe13 2aa0a10501f7 2024-09-13T00:46:37.177 INFO:teuthology.orchestra.run.smithi077.stdout:node-exporter.b smithi088 *:9100 running (9m) 59s ago 17m 8164k - 1.5.0 0da6a335fe13 54d4c3d89eab 2024-09-13T00:46:37.177 INFO:teuthology.orchestra.run.smithi077.stdout:osd.0 smithi077 running (3m) 26s ago 24m 76.6M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 06ef1df834f4 2024-09-13T00:46:37.178 INFO:teuthology.orchestra.run.smithi077.stdout:osd.1 smithi077 running (2m) 26s ago 23m 85.7M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 7db2ad648894 2024-09-13T00:46:37.178 INFO:teuthology.orchestra.run.smithi077.stdout:osd.2 smithi077 running (62s) 26s ago 22m 74.8M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 155fd0391f47 2024-09-13T00:46:37.178 INFO:teuthology.orchestra.run.smithi077.stdout:osd.3 smithi077 running (30s) 26s ago 21m - 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 3786c0a261ed 2024-09-13T00:46:37.178 INFO:teuthology.orchestra.run.smithi077.stdout:osd.4 smithi088 starting - - - 2524M 2024-09-13T00:46:37.178 INFO:teuthology.orchestra.run.smithi077.stdout:osd.5 smithi088 running (19m) 59s ago 19m 122M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb e89a6b298b28 2024-09-13T00:46:37.178 INFO:teuthology.orchestra.run.smithi077.stdout:osd.6 smithi088 running (19m) 59s ago 19m 148M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 599c9665873c 2024-09-13T00:46:37.178 INFO:teuthology.orchestra.run.smithi077.stdout:osd.7 smithi088 running (18m) 59s ago 18m 150M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 0ff0388ffa3c 2024-09-13T00:46:37.178 INFO:teuthology.orchestra.run.smithi077.stdout:prometheus.a smithi088 *:9095 running (8m) 59s ago 17m 58.5M - 2.43.0 a07b618ecd1d 5fb440283869 2024-09-13T00:46:37.251 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:36 smithi088 bash[74245]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T00:46:37.251 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:36 smithi088 bash[74245]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-13T00:46:37.251 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:37 smithi088 bash[74245]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-13T00:46:37.251 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:37 smithi088 bash[74245]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-13T00:46:37.251 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:37 smithi088 bash[74245]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-13T00:46:37.251 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:37 smithi088 bash[74245]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-13T00:46:37.251 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:37 smithi088 bash[74245]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-13T00:46:37.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:37 smithi077 bash[80681]: cluster 2024-09-13T00:46:36.566095+0000 mgr.y (mgr.25304) 320 : cluster [DBG] pgmap v589: 425 pgs: 109 active+undersized, 52 active+undersized+degraded, 264 active+clean; 27 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 661 op/s; 79/564 objects degraded (14.007%) 2024-09-13T00:46:37.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:37 smithi077 bash[80681]: audit 2024-09-13T00:46:37.166426+0000 mgr.y (mgr.25304) 321 : audit [DBG] from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:46:37.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:37 smithi077 bash[83127]: cluster 2024-09-13T00:46:36.566095+0000 mgr.y (mgr.25304) 320 : cluster [DBG] pgmap v589: 425 pgs: 109 active+undersized, 52 active+undersized+degraded, 264 active+clean; 27 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 661 op/s; 79/564 objects degraded (14.007%) 2024-09-13T00:46:37.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:37 smithi077 bash[83127]: audit 2024-09-13T00:46:37.166426+0000 mgr.y (mgr.25304) 321 : audit [DBG] from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:46:37.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:37 smithi088 bash[68960]: cluster 2024-09-13T00:46:36.566095+0000 mgr.y (mgr.25304) 320 : cluster [DBG] pgmap v589: 425 pgs: 109 active+undersized, 52 active+undersized+degraded, 264 active+clean; 27 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.3 MiB/s wr, 661 op/s; 79/564 objects degraded (14.007%) 2024-09-13T00:46:37.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:37 smithi088 bash[68960]: audit 2024-09-13T00:46:37.166426+0000 mgr.y (mgr.25304) 321 : audit [DBG] from='client.34566 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:46:38.079 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph orch ps' 2024-09-13T00:46:38.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:46:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:46:38] "GET /metrics HTTP/1.1" 200 60697 "" "Prometheus/2.43.0" 2024-09-13T00:46:39.124 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.124 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:39.124 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:39.125 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:39.125 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:39.125 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:39.125 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.125 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.125 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:39.125 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:39.125 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:39.125 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:39.125 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:39.125 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.125 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.125 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:39.126 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.126 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.126 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:39.126 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.126 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.126 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:39.126 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:39.126 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:39.126 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:39.126 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:39.126 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.126 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.126 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:39.126 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:39.127 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:39.127 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:39.127 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:39.127 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.127 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.127 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:39.127 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:39.127 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:39.127 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:39.127 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:39.127 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.127 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.127 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:39.128 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:39.128 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:39.128 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:39.128 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:39.128 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.128 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.128 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:39.128 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.128 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.128 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:39.128 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.128 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.128 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:39.129 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:39.129 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:39.129 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:39.129 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:39.129 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.129 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.129 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:39.129 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:39.129 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:39.129 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:39.129 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:39.129 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.130 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.130 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:39.130 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:39.130 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:39.130 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:39.130 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:39.130 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.130 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.130 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:39.130 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:39.130 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:39.130 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:39.130 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:39.130 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.131 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.131 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:39.131 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.131 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.131 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:39.131 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.131 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.131 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:39.131 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:39.131 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:39.131 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:39.131 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:39.131 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.131 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.132 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:39.132 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:39.132 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:39.132 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:39.132 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:39.132 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.132 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.132 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:39.132 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:39.132 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:39.132 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:39.132 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:39.132 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.133 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.133 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:39.133 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:39.133 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:39.133 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:39.133 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:39.133 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.133 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.133 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:39.133 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:39.133 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:39.133 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:39.133 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:39.134 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:39.134 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:39.134 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:39.134 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 4 2024-09-13T00:46:39.134 INFO:tasks.workunit.client.0.smithi077.stdout:read: 4 2024-09-13T00:46:39.134 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS2 (16408 ms) 2024-09-13T00:46:39.134 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestIOWithIOHint 2024-09-13T00:46:39.134 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:39.350 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:38 smithi088 bash[74473]: debug 2024-09-13T00:46:38.924+0000 7f017a7de640 1 -- 172.21.15.88:0/3048552661 <== mon.1 v2:172.21.15.88:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5560ee1931e0 con 0x5560ee153c00 2024-09-13T00:46:39.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:39 smithi088 bash[68960]: cluster 2024-09-13T00:46:38.567478+0000 mgr.y (mgr.25304) 322 : cluster [DBG] pgmap v590: 425 pgs: 109 active+undersized, 52 active+undersized+degraded, 264 active+clean; 27 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.3 MiB/s wr, 550 op/s; 79/564 objects degraded (14.007%) 2024-09-13T00:46:39.851 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:39 smithi088 bash[74473]: debug 2024-09-13T00:46:39.464+0000 7f017d044640 -1 Falling back to public interface 2024-09-13T00:46:40.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:39 smithi077 bash[80681]: cluster 2024-09-13T00:46:38.567478+0000 mgr.y (mgr.25304) 322 : cluster [DBG] pgmap v590: 425 pgs: 109 active+undersized, 52 active+undersized+degraded, 264 active+clean; 27 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.3 MiB/s wr, 550 op/s; 79/564 objects degraded (14.007%) 2024-09-13T00:46:40.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:39 smithi077 bash[83127]: cluster 2024-09-13T00:46:38.567478+0000 mgr.y (mgr.25304) 322 : cluster [DBG] pgmap v590: 425 pgs: 109 active+undersized, 52 active+undersized+degraded, 264 active+clean; 27 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.3 MiB/s rd, 1.3 MiB/s wr, 550 op/s; 79/564 objects degraded (14.007%) 2024-09-13T00:46:41.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:40 smithi077 bash[80681]: cluster 2024-09-13T00:46:40.623038+0000 mon.a (mon.0) 998 : cluster [WRN] Health check update: Degraded data redundancy: 79/573 objects degraded (13.787%), 52 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:41.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:40 smithi077 bash[83127]: cluster 2024-09-13T00:46:40.623038+0000 mon.a (mon.0) 998 : cluster [WRN] Health check update: Degraded data redundancy: 79/573 objects degraded (13.787%), 52 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:41.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:40 smithi088 bash[68960]: cluster 2024-09-13T00:46:40.623038+0000 mon.a (mon.0) 998 : cluster [WRN] Health check update: Degraded data redundancy: 79/573 objects degraded (13.787%), 52 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:42.027 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:41 smithi077 bash[80681]: cluster 2024-09-13T00:46:40.570741+0000 mgr.y (mgr.25304) 323 : cluster [DBG] pgmap v591: 425 pgs: 109 active+undersized, 52 active+undersized+degraded, 264 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 853 op/s; 79/573 objects degraded (13.787%) 2024-09-13T00:46:42.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:41 smithi077 bash[83127]: cluster 2024-09-13T00:46:40.570741+0000 mgr.y (mgr.25304) 323 : cluster [DBG] pgmap v591: 425 pgs: 109 active+undersized, 52 active+undersized+degraded, 264 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 853 op/s; 79/573 objects degraded (13.787%) 2024-09-13T00:46:42.100 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:41 smithi088 bash[68960]: cluster 2024-09-13T00:46:40.570741+0000 mgr.y (mgr.25304) 323 : cluster [DBG] pgmap v591: 425 pgs: 109 active+undersized, 52 active+undersized+degraded, 264 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 1.3 MiB/s wr, 853 op/s; 79/573 objects degraded (13.787%) 2024-09-13T00:46:42.600 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:46:42 smithi088 bash[61791]: ts=2024-09-13T00:46:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:46:42.865 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:46:43.101 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:42 smithi088 bash[74473]: debug 2024-09-13T00:46:42.604+0000 7f017d044640 -1 osd.4 645 log_to_monitors true 2024-09-13T00:46:43.622 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:43.622 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:43.622 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:43.622 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:43.622 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:43.623 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.623 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:43.623 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.623 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:43.623 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:43.623 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.623 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:43.623 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.623 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:43.623 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:43.623 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.623 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:43.624 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.624 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:43.624 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:43.624 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.624 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:43.624 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.624 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:43.624 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:43.624 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.624 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:43.624 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.624 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:43.624 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:43.625 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:43.625 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:43.625 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:43.625 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:43.625 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:43.625 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:43.625 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:43.625 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:43.625 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:43.625 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:43.625 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.625 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:43.625 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.626 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:43.626 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:43.626 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.626 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:43.626 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.626 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:43.626 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:43.626 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.626 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:43.626 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.626 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:43.626 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:43.626 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.626 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:43.627 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.627 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:43.627 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:43.627 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.627 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:43.627 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.627 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:43.627 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:43.627 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:43.627 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:43.627 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:43.627 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:43.627 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:43.628 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.628 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:43.628 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:43.628 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:43.628 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.628 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:43.628 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:43.628 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:43.628 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.628 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:43.628 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:43.628 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:43.628 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.629 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:43.629 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:43.629 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:43.629 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.629 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:43.629 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:43.629 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:43.629 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 512~512 = 512 2024-09-13T00:46:43.629 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.629 INFO:tasks.workunit.client.0.smithi077.stdout:aio discard: 1536~512 = 0 2024-09-13T00:46:43.629 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:43.629 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:43.630 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:43.630 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:43.630 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:43.630 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 0 2024-09-13T00:46:43.630 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.630 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.630 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 0 2024-09-13T00:46:43.630 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.630 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.630 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 16384 2024-09-13T00:46:43.630 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.630 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.630 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 16384 2024-09-13T00:46:43.630 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.631 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.631 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:43.631 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:43.631 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 49152 2024-09-13T00:46:43.631 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.631 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.631 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 49152 2024-09-13T00:46:43.631 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.631 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.631 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 65536 2024-09-13T00:46:43.631 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.631 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.631 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 65536 2024-09-13T00:46:43.632 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.632 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.632 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:43.632 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:43.632 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 98304 2024-09-13T00:46:43.632 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.632 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.632 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 98304 2024-09-13T00:46:43.632 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.632 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.632 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 114688 2024-09-13T00:46:43.632 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.632 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.633 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 114688 2024-09-13T00:46:43.633 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.633 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.633 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:43.633 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:43.633 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 147456 2024-09-13T00:46:43.633 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.633 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.633 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 147456 2024-09-13T00:46:43.633 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.633 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.633 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 163840 2024-09-13T00:46:43.633 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.634 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.634 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 163840 2024-09-13T00:46:43.634 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.634 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.634 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:43.634 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:43.634 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 196608 2024-09-13T00:46:43.634 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.634 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.634 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 196608 2024-09-13T00:46:43.634 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.634 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.634 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 212992 2024-09-13T00:46:43.635 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.635 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.635 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 212992 2024-09-13T00:46:43.635 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.635 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.635 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:43.635 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:43.635 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.635 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:43.635 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.635 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:43.635 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:43.636 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.636 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.636 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.636 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:43.636 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.636 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:43.636 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:43.636 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.636 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.636 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.637 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:43.637 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.637 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:43.637 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:43.637 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.637 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.637 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.637 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:43.637 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.637 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:43.637 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:43.637 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.637 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.638 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.638 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:43.638 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:43.638 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.638 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:43.638 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:43.638 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.638 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:43.638 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.638 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:43.638 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:43.638 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.639 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.639 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.639 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:43.639 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:43.639 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:43.639 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:43.639 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:43.639 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:43.639 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:43.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:43 smithi077 bash[80681]: audit 2024-09-13T00:46:42.363266+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:43.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:43 smithi077 bash[80681]: audit 2024-09-13T00:46:42.371004+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:43.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:43 smithi077 bash[80681]: audit 2024-09-13T00:46:42.612538+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.88:6800/808774429,v1:172.21.15.88:6801/808774429]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T00:46:43.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:43 smithi077 bash[80681]: audit 2024-09-13T00:46:42.613087+0000 mon.a (mon.0) 1001 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T00:46:43.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:43 smithi077 bash[80681]: audit 2024-09-13T00:46:43.093822+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:43.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:43 smithi077 bash[80681]: audit 2024-09-13T00:46:43.102015+0000 mon.a (mon.0) 1003 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:43.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:43 smithi077 bash[83127]: audit 2024-09-13T00:46:42.363266+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:43.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:43 smithi077 bash[83127]: audit 2024-09-13T00:46:42.371004+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:43.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:43 smithi077 bash[83127]: audit 2024-09-13T00:46:42.612538+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.88:6800/808774429,v1:172.21.15.88:6801/808774429]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T00:46:43.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:43 smithi077 bash[83127]: audit 2024-09-13T00:46:42.613087+0000 mon.a (mon.0) 1001 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T00:46:43.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:43 smithi077 bash[83127]: audit 2024-09-13T00:46:43.093822+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:43.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:43 smithi077 bash[83127]: audit 2024-09-13T00:46:43.102015+0000 mon.a (mon.0) 1003 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:43.850 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 00:46:43 smithi088 bash[74473]: debug 2024-09-13T00:46:43.424+0000 7f0174dee640 -1 osd.4 645 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-13T00:46:43.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:43 smithi088 bash[68960]: audit 2024-09-13T00:46:42.363266+0000 mon.a (mon.0) 999 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:43.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:43 smithi088 bash[68960]: audit 2024-09-13T00:46:42.371004+0000 mon.a (mon.0) 1000 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:43.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:43 smithi088 bash[68960]: audit 2024-09-13T00:46:42.612538+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.88:6800/808774429,v1:172.21.15.88:6801/808774429]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T00:46:43.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:43 smithi088 bash[68960]: audit 2024-09-13T00:46:42.613087+0000 mon.a (mon.0) 1001 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-13T00:46:43.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:43 smithi088 bash[68960]: audit 2024-09-13T00:46:43.093822+0000 mon.a (mon.0) 1002 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:43.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:43 smithi088 bash[68960]: audit 2024-09-13T00:46:43.102015+0000 mon.a (mon.0) 1003 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:44.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:44 smithi077 bash[80681]: cluster 2024-09-13T00:46:42.572083+0000 mgr.y (mgr.25304) 324 : cluster [DBG] pgmap v592: 425 pgs: 109 active+undersized, 52 active+undersized+degraded, 264 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.2 MiB/s wr, 971 op/s; 79/576 objects degraded (13.715%) 2024-09-13T00:46:44.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:44 smithi077 bash[80681]: audit 2024-09-13T00:46:43.387325+0000 mon.a (mon.0) 1004 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T00:46:44.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:44 smithi077 bash[80681]: cluster 2024-09-13T00:46:43.387649+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e648: 8 total, 7 up, 8 in 2024-09-13T00:46:44.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:44 smithi077 bash[80681]: audit 2024-09-13T00:46:43.388129+0000 mon.b (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.88:6800/808774429,v1:172.21.15.88:6801/808774429]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:46:44.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:44 smithi077 bash[80681]: audit 2024-09-13T00:46:43.390804+0000 mon.a (mon.0) 1006 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:46:44.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:44 smithi077 bash[80681]: audit 2024-09-13T00:46:43.424753+0000 mon.a (mon.0) 1007 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:44.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:44 smithi077 bash[80681]: audit 2024-09-13T00:46:43.426839+0000 mon.a (mon.0) 1008 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:44.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:44 smithi077 bash[80681]: debug 2024-09-13T00:46:44.415+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:46:44.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:44 smithi077 bash[83127]: cluster 2024-09-13T00:46:42.572083+0000 mgr.y (mgr.25304) 324 : cluster [DBG] pgmap v592: 425 pgs: 109 active+undersized, 52 active+undersized+degraded, 264 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.2 MiB/s wr, 971 op/s; 79/576 objects degraded (13.715%) 2024-09-13T00:46:44.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:44 smithi077 bash[83127]: audit 2024-09-13T00:46:43.387325+0000 mon.a (mon.0) 1004 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T00:46:44.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:44 smithi077 bash[83127]: cluster 2024-09-13T00:46:43.387649+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e648: 8 total, 7 up, 8 in 2024-09-13T00:46:44.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:44 smithi077 bash[83127]: audit 2024-09-13T00:46:43.388129+0000 mon.b (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.88:6800/808774429,v1:172.21.15.88:6801/808774429]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:46:44.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:44 smithi077 bash[83127]: audit 2024-09-13T00:46:43.390804+0000 mon.a (mon.0) 1006 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:46:44.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:44 smithi077 bash[83127]: audit 2024-09-13T00:46:43.424753+0000 mon.a (mon.0) 1007 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:44.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:44 smithi077 bash[83127]: audit 2024-09-13T00:46:43.426839+0000 mon.a (mon.0) 1008 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:44 smithi088 bash[68960]: cluster 2024-09-13T00:46:42.572083+0000 mgr.y (mgr.25304) 324 : cluster [DBG] pgmap v592: 425 pgs: 109 active+undersized, 52 active+undersized+degraded, 264 active+clean; 31 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 1.2 MiB/s wr, 971 op/s; 79/576 objects degraded (13.715%) 2024-09-13T00:46:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:44 smithi088 bash[68960]: audit 2024-09-13T00:46:43.387325+0000 mon.a (mon.0) 1004 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-13T00:46:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:44 smithi088 bash[68960]: cluster 2024-09-13T00:46:43.387649+0000 mon.a (mon.0) 1005 : cluster [DBG] osdmap e648: 8 total, 7 up, 8 in 2024-09-13T00:46:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:44 smithi088 bash[68960]: audit 2024-09-13T00:46:43.388129+0000 mon.b (mon.1) 5 : audit [INF] from='osd.4 [v2:172.21.15.88:6800/808774429,v1:172.21.15.88:6801/808774429]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:46:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:44 smithi088 bash[68960]: audit 2024-09-13T00:46:43.390804+0000 mon.a (mon.0) 1006 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi088", "root=default"]}]: dispatch 2024-09-13T00:46:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:44 smithi088 bash[68960]: audit 2024-09-13T00:46:43.424753+0000 mon.a (mon.0) 1007 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:44.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:44 smithi088 bash[68960]: audit 2024-09-13T00:46:43.426839+0000 mon.a (mon.0) 1008 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:45.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:45 smithi077 bash[83127]: cluster 2024-09-13T00:46:44.422227+0000 mon.a (mon.0) 1009 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:46:45.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:45 smithi077 bash[83127]: cluster 2024-09-13T00:46:44.422270+0000 mon.a (mon.0) 1010 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T00:46:45.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:45 smithi077 bash[83127]: cluster 2024-09-13T00:46:44.432110+0000 mon.a (mon.0) 1011 : cluster [INF] osd.4 [v2:172.21.15.88:6800/808774429,v1:172.21.15.88:6801/808774429] boot 2024-09-13T00:46:45.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:45 smithi077 bash[83127]: cluster 2024-09-13T00:46:44.432296+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e649: 8 total, 8 up, 8 in 2024-09-13T00:46:45.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:45 smithi077 bash[83127]: audit 2024-09-13T00:46:44.435321+0000 mon.a (mon.0) 1013 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:46:45.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:45 smithi077 bash[80681]: cluster 2024-09-13T00:46:44.422227+0000 mon.a (mon.0) 1009 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:46:45.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:45 smithi077 bash[80681]: cluster 2024-09-13T00:46:44.422270+0000 mon.a (mon.0) 1010 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T00:46:45.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:45 smithi077 bash[80681]: cluster 2024-09-13T00:46:44.432110+0000 mon.a (mon.0) 1011 : cluster [INF] osd.4 [v2:172.21.15.88:6800/808774429,v1:172.21.15.88:6801/808774429] boot 2024-09-13T00:46:45.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:45 smithi077 bash[80681]: cluster 2024-09-13T00:46:44.432296+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e649: 8 total, 8 up, 8 in 2024-09-13T00:46:45.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:45 smithi077 bash[80681]: audit 2024-09-13T00:46:44.435321+0000 mon.a (mon.0) 1013 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:46:45.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:45 smithi088 bash[68960]: cluster 2024-09-13T00:46:44.422227+0000 mon.a (mon.0) 1009 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-09-13T00:46:45.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:45 smithi088 bash[68960]: cluster 2024-09-13T00:46:44.422270+0000 mon.a (mon.0) 1010 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-13T00:46:45.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:45 smithi088 bash[68960]: cluster 2024-09-13T00:46:44.432110+0000 mon.a (mon.0) 1011 : cluster [INF] osd.4 [v2:172.21.15.88:6800/808774429,v1:172.21.15.88:6801/808774429] boot 2024-09-13T00:46:45.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:45 smithi088 bash[68960]: cluster 2024-09-13T00:46:44.432296+0000 mon.a (mon.0) 1012 : cluster [DBG] osdmap e649: 8 total, 8 up, 8 in 2024-09-13T00:46:45.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:45 smithi088 bash[68960]: audit 2024-09-13T00:46:44.435321+0000 mon.a (mon.0) 1013 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-13T00:46:46.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:46 smithi077 bash[83127]: cluster 2024-09-13T00:46:44.572859+0000 mgr.y (mgr.25304) 325 : cluster [DBG] pgmap v595: 425 pgs: 109 active+undersized, 52 active+undersized+degraded, 264 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 294 KiB/s wr, 679 op/s; 80/579 objects degraded (13.817%) 2024-09-13T00:46:46.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:46 smithi077 bash[83127]: cluster 2024-09-13T00:46:45.442380+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e650: 8 total, 8 up, 8 in 2024-09-13T00:46:46.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:46 smithi077 bash[83127]: cluster 2024-09-13T00:46:45.973908+0000 mon.a (mon.0) 1015 : cluster [WRN] Health check update: Degraded data redundancy: 80/579 objects degraded (13.817%), 52 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:46.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:46 smithi077 bash[80681]: cluster 2024-09-13T00:46:44.572859+0000 mgr.y (mgr.25304) 325 : cluster [DBG] pgmap v595: 425 pgs: 109 active+undersized, 52 active+undersized+degraded, 264 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 294 KiB/s wr, 679 op/s; 80/579 objects degraded (13.817%) 2024-09-13T00:46:46.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:46 smithi077 bash[80681]: cluster 2024-09-13T00:46:45.442380+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e650: 8 total, 8 up, 8 in 2024-09-13T00:46:46.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:46 smithi077 bash[80681]: cluster 2024-09-13T00:46:45.973908+0000 mon.a (mon.0) 1015 : cluster [WRN] Health check update: Degraded data redundancy: 80/579 objects degraded (13.817%), 52 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:46.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:46 smithi088 bash[68960]: cluster 2024-09-13T00:46:44.572859+0000 mgr.y (mgr.25304) 325 : cluster [DBG] pgmap v595: 425 pgs: 109 active+undersized, 52 active+undersized+degraded, 264 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 294 KiB/s wr, 679 op/s; 80/579 objects degraded (13.817%) 2024-09-13T00:46:46.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:46 smithi088 bash[68960]: cluster 2024-09-13T00:46:45.442380+0000 mon.a (mon.0) 1014 : cluster [DBG] osdmap e650: 8 total, 8 up, 8 in 2024-09-13T00:46:46.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:46 smithi088 bash[68960]: cluster 2024-09-13T00:46:45.973908+0000 mon.a (mon.0) 1015 : cluster [WRN] Health check update: Degraded data redundancy: 80/579 objects degraded (13.817%), 52 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:46.898 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.898 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:46.898 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:46.899 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:46.899 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:46.899 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:46.899 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.899 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.899 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:46.899 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:46.899 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:46.899 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:46.899 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:46.899 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.899 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.899 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:46.899 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.900 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.900 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:46.900 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.900 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.900 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:46.900 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:46.900 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:46.900 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:46.900 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:46.900 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.900 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.900 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:46.900 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:46.901 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:46.901 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:46.901 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:46.901 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.901 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.901 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:46.901 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:46.901 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:46.901 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:46.901 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:46.901 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.901 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.901 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:46.901 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:46.902 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:46.902 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:46.902 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:46.902 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.902 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.902 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:46.902 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.902 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.902 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:46.902 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.902 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.902 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:46.902 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:46.902 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:46.903 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:46.903 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:46.903 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.903 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.903 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:46.903 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:46.903 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:46.903 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:46.903 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:46.903 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.903 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.903 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:46.903 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:46.904 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:46.904 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:46.904 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:46.904 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.904 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.904 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:46.904 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:46.904 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:46.904 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:46.904 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:46.904 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.904 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.904 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:46.905 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.905 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.905 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:46.905 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.905 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.905 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:46.905 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:46.905 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:46.905 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:46.905 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:46.905 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.905 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.905 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:46.906 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:46.906 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:46.906 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:46.906 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:46.906 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.906 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.906 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:46.906 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:46.906 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:46.906 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:46.906 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:46.906 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.906 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.906 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:46.906 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:46.907 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:46.907 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:46.907 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:46.907 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.907 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.907 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:46.907 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:46.907 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:46.907 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:46.907 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:46.907 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:46.907 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:46.907 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestIOWithIOHint (7774 ms) 2024-09-13T00:46:46.908 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestDataPoolIO 2024-09-13T00:46:48.184 INFO:teuthology.orchestra.run.smithi077.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-13T00:46:48.185 INFO:teuthology.orchestra.run.smithi077.stdout:alertmanager.a smithi077 *:9093,9094 running (9m) 37s ago 16m 14.8M - 0.25.0 c8568f914cd2 49a4b5607300 2024-09-13T00:46:48.185 INFO:teuthology.orchestra.run.smithi077.stdout:grafana.a smithi088 *:3000 running (9m) 5s ago 16m 75.8M - 9.4.7 954c08fa6188 c369fdf58d31 2024-09-13T00:46:48.185 INFO:teuthology.orchestra.run.smithi077.stdout:mds.foo.smithi077.bigbbf smithi077 running (14m) 37s ago 14m 14.7M - 17.2.7-1627-gbbdadfa3 214985049bdb 30b25ab8f2dd 2024-09-13T00:46:48.185 INFO:teuthology.orchestra.run.smithi077.stdout:mds.foo.smithi088.krmowh smithi088 running (14m) 5s ago 14m 19.9M - 17.2.7-1627-gbbdadfa3 214985049bdb ebef6ef73335 2024-09-13T00:46:48.185 INFO:teuthology.orchestra.run.smithi077.stdout:mgr.x smithi088 *:8443,9283,8765 running (8m) 5s ago 25m 427M - 18.2.4-837-g7a853b17 f21ba1eea74e fdfc79d860d1 2024-09-13T00:46:48.185 INFO:teuthology.orchestra.run.smithi077.stdout:mgr.y smithi077 *:8443,9283,8765 running (9m) 37s ago 27m 497M - 18.2.4-837-g7a853b17 f21ba1eea74e d46e77b45675 2024-09-13T00:46:48.185 INFO:teuthology.orchestra.run.smithi077.stdout:mon.a smithi077 running (7m) 37s ago 27m 74.1M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 3edacf970db4 2024-09-13T00:46:48.185 INFO:teuthology.orchestra.run.smithi077.stdout:mon.b smithi088 running (5m) 5s ago 25m 45.7M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e dd7e71fd2ce3 2024-09-13T00:46:48.185 INFO:teuthology.orchestra.run.smithi077.stdout:mon.c smithi077 running (6m) 37s ago 25m 59.8M 2048M 18.2.4-837-g7a853b17 f21ba1eea74e 0a377e5bd304 2024-09-13T00:46:48.185 INFO:teuthology.orchestra.run.smithi077.stdout:node-exporter.a smithi077 *:9100 running (9m) 37s ago 17m 8576k - 1.5.0 0da6a335fe13 2aa0a10501f7 2024-09-13T00:46:48.185 INFO:teuthology.orchestra.run.smithi077.stdout:node-exporter.b smithi088 *:9100 running (9m) 5s ago 17m 8596k - 1.5.0 0da6a335fe13 54d4c3d89eab 2024-09-13T00:46:48.185 INFO:teuthology.orchestra.run.smithi077.stdout:osd.0 smithi077 running (3m) 37s ago 24m 76.6M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 06ef1df834f4 2024-09-13T00:46:48.186 INFO:teuthology.orchestra.run.smithi077.stdout:osd.1 smithi077 running (3m) 37s ago 23m 85.7M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 7db2ad648894 2024-09-13T00:46:48.186 INFO:teuthology.orchestra.run.smithi077.stdout:osd.2 smithi077 running (73s) 37s ago 22m 74.8M 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 155fd0391f47 2024-09-13T00:46:48.186 INFO:teuthology.orchestra.run.smithi077.stdout:osd.3 smithi077 running (41s) 37s ago 21m - 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 3786c0a261ed 2024-09-13T00:46:48.186 INFO:teuthology.orchestra.run.smithi077.stdout:osd.4 smithi088 running (10s) 5s ago 20m - 2524M 18.2.4-837-g7a853b17 f21ba1eea74e 9cd4d5c0d5bf 2024-09-13T00:46:48.186 INFO:teuthology.orchestra.run.smithi077.stdout:osd.5 smithi088 running (20m) 5s ago 20m 126M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb e89a6b298b28 2024-09-13T00:46:48.186 INFO:teuthology.orchestra.run.smithi077.stdout:osd.6 smithi088 running (19m) 5s ago 19m 146M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 599c9665873c 2024-09-13T00:46:48.186 INFO:teuthology.orchestra.run.smithi077.stdout:osd.7 smithi088 running (18m) 5s ago 18m 155M 2524M 17.2.7-1627-gbbdadfa3 214985049bdb 0ff0388ffa3c 2024-09-13T00:46:48.186 INFO:teuthology.orchestra.run.smithi077.stdout:prometheus.a smithi088 *:9095 running (8m) 5s ago 17m 61.3M - 2.43.0 a07b618ecd1d 5fb440283869 2024-09-13T00:46:48.467 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:48.777 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:46:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:46:48] "GET /metrics HTTP/1.1" 200 60554 "" "Prometheus/2.43.0" 2024-09-13T00:46:48.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:48 smithi077 bash[83127]: cluster 2024-09-13T00:46:46.575143+0000 mgr.y (mgr.25304) 326 : cluster [DBG] pgmap v597: 425 pgs: 12 peering, 87 active+undersized, 42 active+undersized+degraded, 284 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 629 KiB/s rd, 315 KiB/s wr, 1.26k op/s; 63/579 objects degraded (10.881%) 2024-09-13T00:46:48.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:48 smithi077 bash[83127]: cluster 2024-09-13T00:46:47.459142+0000 mon.a (mon.0) 1016 : cluster [DBG] osdmap e651: 8 total, 8 up, 8 in 2024-09-13T00:46:48.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:48 smithi077 bash[83127]: audit 2024-09-13T00:46:47.462658+0000 mon.a (mon.0) 1017 : audit [INF] from='client.? 172.21.15.77:0/945357005' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:46:48.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:48 smithi077 bash[80681]: cluster 2024-09-13T00:46:46.575143+0000 mgr.y (mgr.25304) 326 : cluster [DBG] pgmap v597: 425 pgs: 12 peering, 87 active+undersized, 42 active+undersized+degraded, 284 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 629 KiB/s rd, 315 KiB/s wr, 1.26k op/s; 63/579 objects degraded (10.881%) 2024-09-13T00:46:48.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:48 smithi077 bash[80681]: cluster 2024-09-13T00:46:47.459142+0000 mon.a (mon.0) 1016 : cluster [DBG] osdmap e651: 8 total, 8 up, 8 in 2024-09-13T00:46:48.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:48 smithi077 bash[80681]: audit 2024-09-13T00:46:47.462658+0000 mon.a (mon.0) 1017 : audit [INF] from='client.? 172.21.15.77:0/945357005' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:46:48.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:48 smithi088 bash[68960]: cluster 2024-09-13T00:46:46.575143+0000 mgr.y (mgr.25304) 326 : cluster [DBG] pgmap v597: 425 pgs: 12 peering, 87 active+undersized, 42 active+undersized+degraded, 284 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 629 KiB/s rd, 315 KiB/s wr, 1.26k op/s; 63/579 objects degraded (10.881%) 2024-09-13T00:46:48.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:48 smithi088 bash[68960]: cluster 2024-09-13T00:46:47.459142+0000 mon.a (mon.0) 1016 : cluster [DBG] osdmap e651: 8 total, 8 up, 8 in 2024-09-13T00:46:48.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:48 smithi088 bash[68960]: audit 2024-09-13T00:46:47.462658+0000 mon.a (mon.0) 1017 : audit [INF] from='client.? 172.21.15.77:0/945357005' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:46:48.945 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9218306c-7165-11ef-bceb-c7b262605968 -e sha1=7a853b1706d076d58e0da7a5d3b9b9eb36276880 -- bash -c 'ceph versions' 2024-09-13T00:46:49.746 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:49 smithi088 bash[68960]: audit 2024-09-13T00:46:48.153198+0000 mgr.y (mgr.25304) 327 : audit [DBG] from='client.44492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:46:49.746 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:49 smithi088 bash[68960]: audit 2024-09-13T00:46:48.463904+0000 mon.a (mon.0) 1018 : audit [INF] from='client.? 172.21.15.77:0/945357005' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:46:49.746 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:49 smithi088 bash[68960]: cluster 2024-09-13T00:46:48.464468+0000 mon.a (mon.0) 1019 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2024-09-13T00:46:49.747 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:49 smithi088 bash[68960]: cluster 2024-09-13T00:46:48.576371+0000 mgr.y (mgr.25304) 328 : cluster [DBG] pgmap v600: 457 pgs: 5 creating+peering, 27 unknown, 12 peering, 35 active+undersized, 22 active+undersized+degraded, 356 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 654 KiB/s rd, 297 KiB/s wr, 1.33k op/s; 28/579 objects degraded (4.836%); 3 B/s, 0 objects/s recovering 2024-09-13T00:46:49.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:49 smithi077 bash[83127]: audit 2024-09-13T00:46:48.153198+0000 mgr.y (mgr.25304) 327 : audit [DBG] from='client.44492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:46:49.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:49 smithi077 bash[83127]: audit 2024-09-13T00:46:48.463904+0000 mon.a (mon.0) 1018 : audit [INF] from='client.? 172.21.15.77:0/945357005' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:46:49.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:49 smithi077 bash[83127]: cluster 2024-09-13T00:46:48.464468+0000 mon.a (mon.0) 1019 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2024-09-13T00:46:49.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:49 smithi077 bash[83127]: cluster 2024-09-13T00:46:48.576371+0000 mgr.y (mgr.25304) 328 : cluster [DBG] pgmap v600: 457 pgs: 5 creating+peering, 27 unknown, 12 peering, 35 active+undersized, 22 active+undersized+degraded, 356 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 654 KiB/s rd, 297 KiB/s wr, 1.33k op/s; 28/579 objects degraded (4.836%); 3 B/s, 0 objects/s recovering 2024-09-13T00:46:49.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:49 smithi077 bash[80681]: audit 2024-09-13T00:46:48.153198+0000 mgr.y (mgr.25304) 327 : audit [DBG] from='client.44492 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-13T00:46:49.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:49 smithi077 bash[80681]: audit 2024-09-13T00:46:48.463904+0000 mon.a (mon.0) 1018 : audit [INF] from='client.? 172.21.15.77:0/945357005' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:46:49.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:49 smithi077 bash[80681]: cluster 2024-09-13T00:46:48.464468+0000 mon.a (mon.0) 1019 : cluster [DBG] osdmap e652: 8 total, 8 up, 8 in 2024-09-13T00:46:49.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:49 smithi077 bash[80681]: cluster 2024-09-13T00:46:48.576371+0000 mgr.y (mgr.25304) 328 : cluster [DBG] pgmap v600: 457 pgs: 5 creating+peering, 27 unknown, 12 peering, 35 active+undersized, 22 active+undersized+degraded, 356 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 654 KiB/s rd, 297 KiB/s wr, 1.33k op/s; 28/579 objects degraded (4.836%); 3 B/s, 0 objects/s recovering 2024-09-13T00:46:50.629 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:50.630 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:50.630 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:50.630 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:50.630 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:50.630 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:50.630 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:50.630 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:50.630 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:50.630 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:50.630 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:50.630 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:50.630 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:50.631 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:50.631 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:50.631 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:50.631 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:50.631 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:50.631 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:50.631 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:50.631 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:50.631 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:50.631 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:50.631 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:50.631 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:50.631 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:50.632 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:50.632 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:50.632 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:50.632 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:50.632 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:50.632 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:50.632 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:50.632 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:50.632 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:50.632 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:50.632 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:50.632 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:50.632 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:50.633 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:50.633 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:50.633 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:50.633 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:50.633 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:50.633 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:50.633 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:50.633 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:50.633 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:50.633 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:50.633 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:50.633 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:50.633 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:50.634 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:50.634 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:50.634 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:50.634 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 512~512 = 512 2024-09-13T00:46:50.634 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:50.634 INFO:tasks.workunit.client.0.smithi077.stdout:aio discard: 1536~512 = 0 2024-09-13T00:46:50.634 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:50.634 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:50.634 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:50.634 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:50.634 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:50.634 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:50.634 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:50.635 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestDataPoolIO (3731 ms) 2024-09-13T00:46:50.635 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatch 2024-09-13T00:46:50.635 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:50.699 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatch (69 ms) 2024-09-13T00:46:50.699 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatch 2024-09-13T00:46:50.699 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:50.768 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatch (69 ms) 2024-09-13T00:46:50.768 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccess 2024-09-13T00:46:50.768 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:50.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:50 smithi077 bash[80681]: cluster 2024-09-13T00:46:49.489750+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2024-09-13T00:46:50.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:50 smithi077 bash[83127]: cluster 2024-09-13T00:46:49.489750+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2024-09-13T00:46:50.832 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccess (65 ms) 2024-09-13T00:46:50.832 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccess 2024-09-13T00:46:50.832 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:50.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:50 smithi088 bash[68960]: cluster 2024-09-13T00:46:49.489750+0000 mon.a (mon.0) 1020 : cluster [DBG] osdmap e653: 8 total, 8 up, 8 in 2024-09-13T00:46:50.899 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccess (67 ms) 2024-09-13T00:46:50.900 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned 2024-09-13T00:46:50.900 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:51.037 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned (137 ms) 2024-09-13T00:46:51.037 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned 2024-09-13T00:46:51.037 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:51.171 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned (135 ms) 2024-09-13T00:46:51.171 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLarge 2024-09-13T00:46:51.171 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:51.289 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLarge (117 ms) 2024-09-13T00:46:51.289 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLarge 2024-09-13T00:46:51.289 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:51.406 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLarge (117 ms) 2024-09-13T00:46:51.406 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess 2024-09-13T00:46:51.406 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:51.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:51 smithi077 bash[80681]: cluster 2024-09-13T00:46:50.490667+0000 mon.a (mon.0) 1021 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2024-09-13T00:46:51.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:51 smithi077 bash[80681]: cluster 2024-09-13T00:46:50.577457+0000 mgr.y (mgr.25304) 329 : cluster [DBG] pgmap v603: 457 pgs: 5 creating+peering, 1 unknown, 451 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 0 B/s wr, 0 op/s; 3 B/s, 0 objects/s recovering 2024-09-13T00:46:51.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:51 smithi077 bash[80681]: cluster 2024-09-13T00:46:50.975207+0000 mon.a (mon.0) 1022 : cluster [WRN] Health check update: Degraded data redundancy: 28/579 objects degraded (4.836%), 22 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:51.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:51 smithi077 bash[83127]: cluster 2024-09-13T00:46:50.490667+0000 mon.a (mon.0) 1021 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2024-09-13T00:46:51.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:51 smithi077 bash[83127]: cluster 2024-09-13T00:46:50.577457+0000 mgr.y (mgr.25304) 329 : cluster [DBG] pgmap v603: 457 pgs: 5 creating+peering, 1 unknown, 451 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 0 B/s wr, 0 op/s; 3 B/s, 0 objects/s recovering 2024-09-13T00:46:51.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:51 smithi077 bash[83127]: cluster 2024-09-13T00:46:50.975207+0000 mon.a (mon.0) 1022 : cluster [WRN] Health check update: Degraded data redundancy: 28/579 objects degraded (4.836%), 22 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:51.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:51 smithi088 bash[68960]: cluster 2024-09-13T00:46:50.490667+0000 mon.a (mon.0) 1021 : cluster [DBG] osdmap e654: 8 total, 8 up, 8 in 2024-09-13T00:46:51.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:51 smithi088 bash[68960]: cluster 2024-09-13T00:46:50.577457+0000 mgr.y (mgr.25304) 329 : cluster [DBG] pgmap v603: 457 pgs: 5 creating+peering, 1 unknown, 451 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 0 B/s wr, 0 op/s; 3 B/s, 0 objects/s recovering 2024-09-13T00:46:51.850 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:51 smithi088 bash[68960]: cluster 2024-09-13T00:46:50.975207+0000 mon.a (mon.0) 1022 : cluster [WRN] Health check update: Degraded data redundancy: 28/579 objects degraded (4.836%), 22 pgs degraded (PG_DEGRADED) 2024-09-13T00:46:51.863 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess (457 ms) 2024-09-13T00:46:51.864 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess 2024-09-13T00:46:51.864 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:52.308 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess (445 ms) 2024-09-13T00:46:52.308 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers 2024-09-13T00:46:52.308 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:52.352 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers (43 ms) 2024-09-13T00:46:52.352 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVMismatch 2024-09-13T00:46:52.352 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:52.400 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVMismatch (49 ms) 2024-09-13T00:46:52.400 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVSuccess 2024-09-13T00:46:52.400 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:52.450 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVSuccess (50 ms) 2024-09-13T00:46:52.450 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestScatterGatherIO 2024-09-13T00:46:52.451 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:52.501 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestScatterGatherIO (51 ms) 2024-09-13T00:46:52.502 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestEmptyDiscard 2024-09-13T00:46:52.502 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:52.552 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.552 INFO:tasks.workunit.client.0.smithi077.stdout:aio discard: 0~1048576 = 0 2024-09-13T00:46:52.552 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.552 INFO:tasks.workunit.client.0.smithi077.stdout:aio discard: 0~4194304 = 0 2024-09-13T00:46:52.552 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.552 INFO:tasks.workunit.client.0.smithi077.stdout:aio discard: 3145728~1048576 = 0 2024-09-13T00:46:52.552 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestEmptyDiscard (51 ms) 2024-09-13T00:46:52.552 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestFUA 2024-09-13T00:46:52.553 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:52.597 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:52 smithi088 bash[68960]: cluster 2024-09-13T00:46:51.501774+0000 mon.a (mon.0) 1023 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/579 objects degraded (4.836%), 22 pgs degraded) 2024-09-13T00:46:52.598 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:46:52 smithi088 bash[61791]: ts=2024-09-13T00:46:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:46:52.777 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:52 smithi077 bash[80681]: cluster 2024-09-13T00:46:51.501774+0000 mon.a (mon.0) 1023 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/579 objects degraded (4.836%), 22 pgs degraded) 2024-09-13T00:46:52.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:52 smithi077 bash[83127]: cluster 2024-09-13T00:46:51.501774+0000 mon.a (mon.0) 1023 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 28/579 objects degraded (4.836%), 22 pgs degraded) 2024-09-13T00:46:52.815 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:52.815 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:52.815 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:52.815 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:52.815 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:52.815 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:52.815 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:52.815 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:52.815 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:52.815 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:52.816 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.816 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:52.816 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.816 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:52.816 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:52.816 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.816 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:52.816 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.816 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:52.816 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:52.816 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.816 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:52.816 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.817 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:52.817 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:52.817 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.817 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:52.817 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.817 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:52.817 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:52.817 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.817 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:52.817 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.817 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:52.817 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:52.817 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.817 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:52.817 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:52.818 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:52.818 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.818 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:52.818 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:52.818 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:52.818 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.818 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:52.818 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:52.818 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:52.818 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.818 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:52.818 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:52.818 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:52.818 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.819 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:52.819 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:52.819 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:52.819 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestFUA (262 ms) 2024-09-13T00:46:52.819 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestIOPP 2024-09-13T00:46:52.819 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:46:52.860 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:52.860 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:52.860 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:52.860 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:52.860 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:46:52.860 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.860 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:52.861 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.861 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:52.861 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:52.861 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.861 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:52.861 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.861 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:52.861 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:52.861 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.861 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:52.861 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.861 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:52.861 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:52.861 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.862 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:52.862 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.862 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:52.862 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:52.862 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.862 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:46:52.862 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.893 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:52.893 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:46:52.893 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:52.893 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:52.893 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:52.893 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:52.893 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:52.893 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:52.893 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:52.893 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:52.893 INFO:tasks.workunit.client.0.smithi077.stdout:start compare and write 2024-09-13T00:46:52.894 INFO:tasks.workunit.client.0.smithi077.stdout:compare and wrote: 512 2024-09-13T00:46:52.894 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.894 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:52.894 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.895 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:52.895 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:52.895 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.895 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:52.895 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.898 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:52.898 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:52.898 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.898 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:52.898 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.900 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:52.901 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:52.901 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.901 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:52.901 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.904 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:52.904 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:52.904 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.904 INFO:tasks.workunit.client.0.smithi077.stdout:started aio compare and write 2024-09-13T00:46:52.904 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:52.908 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:52.909 INFO:tasks.workunit.client.0.smithi077.stdout:finished aio compare and write 2024-09-13T00:46:52.909 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:52.909 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:52.909 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:52.909 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:52.909 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:52.909 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.909 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:52.909 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:52.909 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:52.910 INFO:tasks.workunit.client.0.smithi077.stdout:finished read 2024-09-13T00:46:52.910 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.910 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:52.910 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:52.910 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:52.910 INFO:tasks.workunit.client.0.smithi077.stdout:finished read 2024-09-13T00:46:52.910 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.910 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:52.910 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:52.911 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:52.911 INFO:tasks.workunit.client.0.smithi077.stdout:finished read 2024-09-13T00:46:52.912 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.912 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:52.912 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:52.913 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:52.913 INFO:tasks.workunit.client.0.smithi077.stdout:finished read 2024-09-13T00:46:52.913 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:52.913 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:46:52.913 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:46:52.913 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:46:52.913 INFO:tasks.workunit.client.0.smithi077.stdout:finished read 2024-09-13T00:46:52.913 INFO:tasks.workunit.client.0.smithi077.stdout:discard: 512~512 2024-09-13T00:46:52.913 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:53.724 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:46:54.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[80681]: cluster 2024-09-13T00:46:52.579370+0000 mgr.y (mgr.25304) 330 : cluster [DBG] pgmap v604: 457 pgs: 5 creating+peering, 452 active+clean; 37 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 891 KiB/s rd, 843 KiB/s wr, 183 op/s; 3 B/s, 0 objects/s recovering 2024-09-13T00:46:54.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[80681]: audit 2024-09-13T00:46:53.132892+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[80681]: audit 2024-09-13T00:46:53.144001+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[80681]: audit 2024-09-13T00:46:53.146185+0000 mon.a (mon.0) 1026 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:46:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[80681]: audit 2024-09-13T00:46:53.148038+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:46:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[80681]: audit 2024-09-13T00:46:53.158991+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[80681]: audit 2024-09-13T00:46:53.298346+0000 mon.a (mon.0) 1029 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:46:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[80681]: audit 2024-09-13T00:46:53.300403+0000 mon.a (mon.0) 1030 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:46:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[80681]: audit 2024-09-13T00:46:53.301423+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:46:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[80681]: audit 2024-09-13T00:46:53.307877+0000 mon.a (mon.0) 1032 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:54.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[83127]: cluster 2024-09-13T00:46:52.579370+0000 mgr.y (mgr.25304) 330 : cluster [DBG] pgmap v604: 457 pgs: 5 creating+peering, 452 active+clean; 37 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 891 KiB/s rd, 843 KiB/s wr, 183 op/s; 3 B/s, 0 objects/s recovering 2024-09-13T00:46:54.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[83127]: audit 2024-09-13T00:46:53.132892+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:54.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[83127]: audit 2024-09-13T00:46:53.144001+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:54.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[83127]: audit 2024-09-13T00:46:53.146185+0000 mon.a (mon.0) 1026 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:46:54.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[83127]: audit 2024-09-13T00:46:53.148038+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:46:54.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[83127]: audit 2024-09-13T00:46:53.158991+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:54.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[83127]: audit 2024-09-13T00:46:53.298346+0000 mon.a (mon.0) 1029 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:46:54.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[83127]: audit 2024-09-13T00:46:53.300403+0000 mon.a (mon.0) 1030 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:46:54.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[83127]: audit 2024-09-13T00:46:53.301423+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:46:54.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:54 smithi077 bash[83127]: audit 2024-09-13T00:46:53.307877+0000 mon.a (mon.0) 1032 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:54 smithi088 bash[68960]: cluster 2024-09-13T00:46:52.579370+0000 mgr.y (mgr.25304) 330 : cluster [DBG] pgmap v604: 457 pgs: 5 creating+peering, 452 active+clean; 37 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 891 KiB/s rd, 843 KiB/s wr, 183 op/s; 3 B/s, 0 objects/s recovering 2024-09-13T00:46:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:54 smithi088 bash[68960]: audit 2024-09-13T00:46:53.132892+0000 mon.a (mon.0) 1024 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:54 smithi088 bash[68960]: audit 2024-09-13T00:46:53.144001+0000 mon.a (mon.0) 1025 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:54 smithi088 bash[68960]: audit 2024-09-13T00:46:53.146185+0000 mon.a (mon.0) 1026 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:46:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:54 smithi088 bash[68960]: audit 2024-09-13T00:46:53.148038+0000 mon.a (mon.0) 1027 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:46:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:54 smithi088 bash[68960]: audit 2024-09-13T00:46:53.158991+0000 mon.a (mon.0) 1028 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:54 smithi088 bash[68960]: audit 2024-09-13T00:46:53.298346+0000 mon.a (mon.0) 1029 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:46:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:54 smithi088 bash[68960]: audit 2024-09-13T00:46:53.300403+0000 mon.a (mon.0) 1030 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:46:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:54 smithi088 bash[68960]: audit 2024-09-13T00:46:53.301423+0000 mon.a (mon.0) 1031 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:46:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:54 smithi088 bash[68960]: audit 2024-09-13T00:46:53.307877+0000 mon.a (mon.0) 1032 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:56.431 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:56.431 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:56.431 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:56.432 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:56.432 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:46:56.432 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 0 2024-09-13T00:46:56.432 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.432 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.432 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 0 2024-09-13T00:46:56.432 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.432 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.432 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 16384 2024-09-13T00:46:56.432 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.432 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.432 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 16384 2024-09-13T00:46:56.432 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.432 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.433 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:56.433 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:56.433 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 49152 2024-09-13T00:46:56.433 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.433 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.433 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 49152 2024-09-13T00:46:56.433 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.433 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.433 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 65536 2024-09-13T00:46:56.433 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.433 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.433 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 65536 2024-09-13T00:46:56.433 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.434 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.434 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:56.434 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:56.434 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 98304 2024-09-13T00:46:56.434 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.434 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.434 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 98304 2024-09-13T00:46:56.434 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.434 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.434 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 114688 2024-09-13T00:46:56.434 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.434 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.434 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 114688 2024-09-13T00:46:56.434 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.435 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.435 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:56.435 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:56.435 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 147456 2024-09-13T00:46:56.435 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.435 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.435 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 147456 2024-09-13T00:46:56.435 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.435 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.435 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 163840 2024-09-13T00:46:56.435 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.435 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.435 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 163840 2024-09-13T00:46:56.435 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.435 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.436 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:56.436 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:56.436 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 196608 2024-09-13T00:46:56.436 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.436 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.436 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 196608 2024-09-13T00:46:56.436 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.436 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.436 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 212992 2024-09-13T00:46:56.436 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.436 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.436 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 212992 2024-09-13T00:46:56.436 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.437 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.437 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:56.437 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:56.437 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:56.437 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:56.437 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:56.437 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:56.437 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:56.437 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.437 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.437 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:56.437 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:56.437 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:56.437 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:56.438 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:56.438 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.438 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.438 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:56.438 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:56.438 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:56.457 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:56.457 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:56.457 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.457 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.457 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:56.457 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:56.457 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:56.482 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:56.482 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:56.482 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.482 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.482 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:56.482 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:56.482 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:56.482 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:56.482 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:56.482 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:56.483 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:56.483 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:56.483 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:56.496 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:56 smithi077 bash[80681]: cluster 2024-09-13T00:46:54.582787+0000 mgr.y (mgr.25304) 331 : cluster [DBG] pgmap v605: 457 pgs: 457 active+clean; 75 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 7.3 MiB/s wr, 631 op/s 2024-09-13T00:46:56.496 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:56 smithi077 bash[83127]: cluster 2024-09-13T00:46:54.582787+0000 mgr.y (mgr.25304) 331 : cluster [DBG] pgmap v605: 457 pgs: 457 active+clean; 75 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 7.3 MiB/s wr, 631 op/s 2024-09-13T00:46:56.551 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:56.551 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:56.551 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.552 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.552 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:56.552 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:56.552 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:56.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:56 smithi088 bash[68960]: cluster 2024-09-13T00:46:54.582787+0000 mgr.y (mgr.25304) 331 : cluster [DBG] pgmap v605: 457 pgs: 457 active+clean; 75 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 7.3 MiB/s wr, 631 op/s 2024-09-13T00:46:56.624 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:56.624 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:56.624 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.624 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.624 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:56.624 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:56.624 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:56.738 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:56.738 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:56.738 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.738 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.739 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:56.739 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:56.739 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:56.861 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:56.861 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:56.861 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:56.861 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:56.861 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:56.861 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:56.861 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:56.861 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:56.861 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:56.861 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:56.861 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:56.861 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:56.862 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:57.019 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:57.019 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:57.019 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:57.019 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:57.019 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:57.020 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:57.020 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:57.210 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:57.210 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:57.210 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:57.210 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:57.210 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:57.211 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:57.211 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:57.429 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:57.429 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:57.429 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:57.429 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:57.429 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:57.429 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:57.429 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:57.604 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:57.604 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:57.604 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:57.604 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:57.604 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:57.604 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:57.604 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:57.605 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:57.605 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:57.605 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:57.605 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:57.605 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:57.605 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:57.817 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:57.817 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:57.817 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:57.817 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:57.818 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:57.818 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:57.818 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:58.038 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:58.039 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:58.039 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:58.039 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:58.039 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:58.039 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:58.039 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:58.107 INFO:teuthology.orchestra.run.smithi077.stdout:{ 2024-09-13T00:46:58.107 INFO:teuthology.orchestra.run.smithi077.stdout: "mon": { 2024-09-13T00:46:58.107 INFO:teuthology.orchestra.run.smithi077.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 3 2024-09-13T00:46:58.107 INFO:teuthology.orchestra.run.smithi077.stdout: }, 2024-09-13T00:46:58.107 INFO:teuthology.orchestra.run.smithi077.stdout: "mgr": { 2024-09-13T00:46:58.107 INFO:teuthology.orchestra.run.smithi077.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 2 2024-09-13T00:46:58.107 INFO:teuthology.orchestra.run.smithi077.stdout: }, 2024-09-13T00:46:58.107 INFO:teuthology.orchestra.run.smithi077.stdout: "osd": { 2024-09-13T00:46:58.107 INFO:teuthology.orchestra.run.smithi077.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 3, 2024-09-13T00:46:58.108 INFO:teuthology.orchestra.run.smithi077.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 5 2024-09-13T00:46:58.108 INFO:teuthology.orchestra.run.smithi077.stdout: }, 2024-09-13T00:46:58.108 INFO:teuthology.orchestra.run.smithi077.stdout: "mds": { 2024-09-13T00:46:58.108 INFO:teuthology.orchestra.run.smithi077.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 2 2024-09-13T00:46:58.108 INFO:teuthology.orchestra.run.smithi077.stdout: }, 2024-09-13T00:46:58.108 INFO:teuthology.orchestra.run.smithi077.stdout: "overall": { 2024-09-13T00:46:58.108 INFO:teuthology.orchestra.run.smithi077.stdout: "ceph version 17.2.7-1627-gbbdadfa3 (bbdadfa31f8fb9afaa389358d138cac49bd69e56) quincy (stable)": 5, 2024-09-13T00:46:58.108 INFO:teuthology.orchestra.run.smithi077.stdout: "ceph version 18.2.4-837-g7a853b17 (7a853b1706d076d58e0da7a5d3b9b9eb36276880) reef (stable)": 10 2024-09-13T00:46:58.108 INFO:teuthology.orchestra.run.smithi077.stdout: } 2024-09-13T00:46:58.108 INFO:teuthology.orchestra.run.smithi077.stdout:} 2024-09-13T00:46:58.259 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:58.259 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:58.259 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:58.259 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:58.259 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:58.260 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:58.260 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:58.370 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:58 smithi077 bash[83127]: cluster 2024-09-13T00:46:56.584022+0000 mgr.y (mgr.25304) 332 : cluster [DBG] pgmap v606: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 7.1 MiB/s wr, 509 op/s 2024-09-13T00:46:58.371 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:58 smithi077 bash[83127]: audit 2024-09-13T00:46:58.109789+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.77:0/1368790450' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:46:58.371 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:58 smithi077 bash[80681]: cluster 2024-09-13T00:46:56.584022+0000 mgr.y (mgr.25304) 332 : cluster [DBG] pgmap v606: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 7.1 MiB/s wr, 509 op/s 2024-09-13T00:46:58.371 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:58 smithi077 bash[80681]: audit 2024-09-13T00:46:58.109789+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.77:0/1368790450' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:46:58.498 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:58.498 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:58.498 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:58.498 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:58.498 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:58.498 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:58.499 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:58.499 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:58.499 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:58.499 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:58.499 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:58.499 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:58.499 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:58.600 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:58 smithi088 bash[68960]: cluster 2024-09-13T00:46:56.584022+0000 mgr.y (mgr.25304) 332 : cluster [DBG] pgmap v606: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 7.1 MiB/s wr, 509 op/s 2024-09-13T00:46:58.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:58 smithi088 bash[68960]: audit 2024-09-13T00:46:58.109789+0000 mon.c (mon.2) 45 : audit [DBG] from='client.? 172.21.15.77:0/1368790450' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-13T00:46:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:46:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:46:58] "GET /metrics HTTP/1.1" 200 63096 "" "Prometheus/2.43.0" 2024-09-13T00:46:58.800 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:58.801 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:58.801 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:58.801 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:58.801 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:58.801 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:58.801 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:58.967 DEBUG:teuthology.parallel:result is None 2024-09-13T00:46:59.150 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:59.150 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:59.150 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:59.150 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:59.151 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:59.151 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:59.151 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:59.566 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:59.566 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:59.566 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:59.566 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:59.566 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:59.566 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:59.567 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:46:59.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:59 smithi077 bash[83127]: audit 2024-09-13T00:46:58.428861+0000 mon.a (mon.0) 1033 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:59.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:46:59 smithi077 bash[83127]: audit 2024-09-13T00:46:58.430935+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:59.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:59 smithi077 bash[80681]: audit 2024-09-13T00:46:58.428861+0000 mon.a (mon.0) 1033 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:59.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:46:59 smithi077 bash[80681]: audit 2024-09-13T00:46:58.430935+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:59.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:59 smithi088 bash[68960]: audit 2024-09-13T00:46:58.428861+0000 mon.a (mon.0) 1033 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:46:59.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:46:59 smithi088 bash[68960]: audit 2024-09-13T00:46:58.430935+0000 mon.a (mon.0) 1034 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:46:59.951 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:46:59.951 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:46:59.952 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:46:59.952 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:46:59.952 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:59.952 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:59.952 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:59.952 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:46:59.952 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:46:59.952 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:46:59.952 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestIOPP (7137 ms) 2024-09-13T00:46:59.952 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP 2024-09-13T00:46:59.952 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:00.014 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP (63 ms) 2024-09-13T00:47:00.015 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP 2024-09-13T00:47:00.015 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:00.072 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP (58 ms) 2024-09-13T00:47:00.073 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP 2024-09-13T00:47:00.073 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:00.134 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP (62 ms) 2024-09-13T00:47:00.135 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP 2024-09-13T00:47:00.135 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:00.195 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP (60 ms) 2024-09-13T00:47:00.195 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchPP 2024-09-13T00:47:00.195 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:00.260 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchPP (66 ms) 2024-09-13T00:47:00.261 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchPP 2024-09-13T00:47:00.261 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:00.317 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:00.326 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchPP (66 ms) 2024-09-13T00:47:00.326 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP 2024-09-13T00:47:00.326 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:00.394 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP (68 ms) 2024-09-13T00:47:00.394 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP 2024-09-13T00:47:00.394 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:00.488 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:00.497 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP (102 ms) 2024-09-13T00:47:00.497 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessPP 2024-09-13T00:47:00.497 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:00.550 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessPP (53 ms) 2024-09-13T00:47:00.550 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessPP 2024-09-13T00:47:00.550 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:00.602 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:00.611 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessPP (62 ms) 2024-09-13T00:47:00.611 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP 2024-09-13T00:47:00.611 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:00.676 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP (64 ms) 2024-09-13T00:47:00.676 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP 2024-09-13T00:47:00.676 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:00.736 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:00.745 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP (70 ms) 2024-09-13T00:47:00.746 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP 2024-09-13T00:47:00.746 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:00.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:00 smithi077 bash[80681]: cluster 2024-09-13T00:46:58.585966+0000 mgr.y (mgr.25304) 333 : cluster [DBG] pgmap v607: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 6.2 MiB/s wr, 448 op/s 2024-09-13T00:47:00.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:00 smithi077 bash[80681]: audit 2024-09-13T00:47:00.005452+0000 mon.a (mon.0) 1035 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.d", "id": [0, 2]}]: dispatch 2024-09-13T00:47:00.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:00 smithi077 bash[80681]: audit 2024-09-13T00:47:00.006203+0000 mon.a (mon.0) 1036 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]}]: dispatch 2024-09-13T00:47:00.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:00 smithi077 bash[80681]: audit 2024-09-13T00:47:00.006794+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.f", "id": [6, 4]}]: dispatch 2024-09-13T00:47:00.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:00 smithi077 bash[80681]: audit 2024-09-13T00:47:00.007284+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.3", "id": [6, 5]}]: dispatch 2024-09-13T00:47:00.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:00 smithi077 bash[80681]: audit 2024-09-13T00:47:00.007766+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "19.11"}]: dispatch 2024-09-13T00:47:00.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:00 smithi077 bash[80681]: audit 2024-09-13T00:47:00.008239+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "19.1d"}]: dispatch 2024-09-13T00:47:00.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:00 smithi077 bash[83127]: cluster 2024-09-13T00:46:58.585966+0000 mgr.y (mgr.25304) 333 : cluster [DBG] pgmap v607: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 6.2 MiB/s wr, 448 op/s 2024-09-13T00:47:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:00 smithi077 bash[83127]: audit 2024-09-13T00:47:00.005452+0000 mon.a (mon.0) 1035 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.d", "id": [0, 2]}]: dispatch 2024-09-13T00:47:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:00 smithi077 bash[83127]: audit 2024-09-13T00:47:00.006203+0000 mon.a (mon.0) 1036 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]}]: dispatch 2024-09-13T00:47:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:00 smithi077 bash[83127]: audit 2024-09-13T00:47:00.006794+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.f", "id": [6, 4]}]: dispatch 2024-09-13T00:47:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:00 smithi077 bash[83127]: audit 2024-09-13T00:47:00.007284+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.3", "id": [6, 5]}]: dispatch 2024-09-13T00:47:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:00 smithi077 bash[83127]: audit 2024-09-13T00:47:00.007766+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "19.11"}]: dispatch 2024-09-13T00:47:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:00 smithi077 bash[83127]: audit 2024-09-13T00:47:00.008239+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "19.1d"}]: dispatch 2024-09-13T00:47:00.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:00 smithi088 bash[68960]: cluster 2024-09-13T00:46:58.585966+0000 mgr.y (mgr.25304) 333 : cluster [DBG] pgmap v607: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 6.2 MiB/s wr, 448 op/s 2024-09-13T00:47:00.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:00 smithi088 bash[68960]: audit 2024-09-13T00:47:00.005452+0000 mon.a (mon.0) 1035 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.d", "id": [0, 2]}]: dispatch 2024-09-13T00:47:00.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:00 smithi088 bash[68960]: audit 2024-09-13T00:47:00.006203+0000 mon.a (mon.0) 1036 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]}]: dispatch 2024-09-13T00:47:00.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:00 smithi088 bash[68960]: audit 2024-09-13T00:47:00.006794+0000 mon.a (mon.0) 1037 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.f", "id": [6, 4]}]: dispatch 2024-09-13T00:47:00.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:00 smithi088 bash[68960]: audit 2024-09-13T00:47:00.007284+0000 mon.a (mon.0) 1038 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.3", "id": [6, 5]}]: dispatch 2024-09-13T00:47:00.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:00 smithi088 bash[68960]: audit 2024-09-13T00:47:00.007766+0000 mon.a (mon.0) 1039 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "19.11"}]: dispatch 2024-09-13T00:47:00.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:00 smithi088 bash[68960]: audit 2024-09-13T00:47:00.008239+0000 mon.a (mon.0) 1040 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "19.1d"}]: dispatch 2024-09-13T00:47:00.897 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP (151 ms) 2024-09-13T00:47:00.897 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP 2024-09-13T00:47:00.897 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:00.991 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:01.055 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP (158 ms) 2024-09-13T00:47:01.055 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLargePP 2024-09-13T00:47:01.055 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:01.173 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLargePP (118 ms) 2024-09-13T00:47:01.173 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLargePP 2024-09-13T00:47:01.173 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:01.266 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:01.320 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLargePP (148 ms) 2024-09-13T00:47:01.320 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP 2024-09-13T00:47:01.320 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:01.725 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP (405 ms) 2024-09-13T00:47:01.726 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP 2024-09-13T00:47:01.726 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:01 smithi077 bash[80681]: audit 2024-09-13T00:47:00.441888+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.d", "id": [0, 2]}]': finished 2024-09-13T00:47:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:01 smithi077 bash[80681]: audit 2024-09-13T00:47:00.442579+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]}]': finished 2024-09-13T00:47:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:01 smithi077 bash[80681]: audit 2024-09-13T00:47:00.442746+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.f", "id": [6, 4]}]': finished 2024-09-13T00:47:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:01 smithi077 bash[80681]: audit 2024-09-13T00:47:00.442867+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.3", "id": [6, 5]}]': finished 2024-09-13T00:47:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:01 smithi077 bash[80681]: audit 2024-09-13T00:47:00.442950+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "19.11"}]': finished 2024-09-13T00:47:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:01 smithi077 bash[80681]: audit 2024-09-13T00:47:00.443067+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "19.1d"}]': finished 2024-09-13T00:47:01.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:01 smithi077 bash[80681]: cluster 2024-09-13T00:47:00.443177+0000 mon.a (mon.0) 1047 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2024-09-13T00:47:01.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:01 smithi077 bash[80681]: cluster 2024-09-13T00:47:01.000070+0000 mon.a (mon.0) 1048 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2024-09-13T00:47:01.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:01 smithi077 bash[83127]: audit 2024-09-13T00:47:00.441888+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.d", "id": [0, 2]}]': finished 2024-09-13T00:47:01.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:01 smithi077 bash[83127]: audit 2024-09-13T00:47:00.442579+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]}]': finished 2024-09-13T00:47:01.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:01 smithi077 bash[83127]: audit 2024-09-13T00:47:00.442746+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.f", "id": [6, 4]}]': finished 2024-09-13T00:47:01.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:01 smithi077 bash[83127]: audit 2024-09-13T00:47:00.442867+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.3", "id": [6, 5]}]': finished 2024-09-13T00:47:01.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:01 smithi077 bash[83127]: audit 2024-09-13T00:47:00.442950+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "19.11"}]': finished 2024-09-13T00:47:01.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:01 smithi077 bash[83127]: audit 2024-09-13T00:47:00.443067+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "19.1d"}]': finished 2024-09-13T00:47:01.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:01 smithi077 bash[83127]: cluster 2024-09-13T00:47:00.443177+0000 mon.a (mon.0) 1047 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2024-09-13T00:47:01.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:01 smithi077 bash[83127]: cluster 2024-09-13T00:47:01.000070+0000 mon.a (mon.0) 1048 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2024-09-13T00:47:01.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:01 smithi088 bash[68960]: audit 2024-09-13T00:47:00.441888+0000 mon.a (mon.0) 1041 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.d", "id": [0, 2]}]': finished 2024-09-13T00:47:01.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:01 smithi088 bash[68960]: audit 2024-09-13T00:47:00.442579+0000 mon.a (mon.0) 1042 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]}]': finished 2024-09-13T00:47:01.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:01 smithi088 bash[68960]: audit 2024-09-13T00:47:00.442746+0000 mon.a (mon.0) 1043 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.f", "id": [6, 4]}]': finished 2024-09-13T00:47:01.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:01 smithi088 bash[68960]: audit 2024-09-13T00:47:00.442867+0000 mon.a (mon.0) 1044 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.3", "id": [6, 5]}]': finished 2024-09-13T00:47:01.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:01 smithi088 bash[68960]: audit 2024-09-13T00:47:00.442950+0000 mon.a (mon.0) 1045 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "19.11"}]': finished 2024-09-13T00:47:01.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:01 smithi088 bash[68960]: audit 2024-09-13T00:47:00.443067+0000 mon.a (mon.0) 1046 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd rm-pg-upmap-items", "format": "json", "pgid": "19.1d"}]': finished 2024-09-13T00:47:01.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:01 smithi088 bash[68960]: cluster 2024-09-13T00:47:00.443177+0000 mon.a (mon.0) 1047 : cluster [DBG] osdmap e655: 8 total, 8 up, 8 in 2024-09-13T00:47:01.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:01 smithi088 bash[68960]: cluster 2024-09-13T00:47:01.000070+0000 mon.a (mon.0) 1048 : cluster [DBG] osdmap e656: 8 total, 8 up, 8 in 2024-09-13T00:47:02.106 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:02.149 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP (424 ms) 2024-09-13T00:47:02.149 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestIOPPWithIOHint 2024-09-13T00:47:02.149 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:02.217 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:47:02.217 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:47:02.217 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:47:02.217 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:47:02.217 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:47:02.217 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:02.217 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:47:02.217 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:02.219 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:02.219 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:47:02.219 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:02.219 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:47:02.219 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:02.221 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:02.221 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:47:02.222 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:02.222 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:47:02.222 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:02.224 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:02.224 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:47:02.224 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:02.224 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:47:02.224 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:02.229 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:02.229 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:47:02.229 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:02.229 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:47:02.229 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:02.231 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:02.231 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:47:02.231 INFO:tasks.workunit.client.0.smithi077.stdout:read: 512 2024-09-13T00:47:02.231 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:02.231 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:47:02.231 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:47:02.231 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:47:02.231 INFO:tasks.workunit.client.0.smithi077.stdout:finished read 2024-09-13T00:47:02.231 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:02.232 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:47:02.232 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:47:02.232 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:47:02.232 INFO:tasks.workunit.client.0.smithi077.stdout:finished read 2024-09-13T00:47:02.232 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:02.232 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:47:02.232 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:47:02.233 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:47:02.233 INFO:tasks.workunit.client.0.smithi077.stdout:finished read 2024-09-13T00:47:02.233 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:02.233 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:47:02.233 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:47:02.233 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:47:02.233 INFO:tasks.workunit.client.0.smithi077.stdout:finished read 2024-09-13T00:47:02.233 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:02.233 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:47:02.234 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:47:02.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:02 smithi088 bash[68960]: cluster 2024-09-13T00:47:00.586785+0000 mgr.y (mgr.25304) 334 : cluster [DBG] pgmap v609: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1k op/s 2024-09-13T00:47:02.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:47:02 smithi088 bash[61791]: ts=2024-09-13T00:47:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:47:02.777 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:02 smithi077 bash[83127]: cluster 2024-09-13T00:47:00.586785+0000 mgr.y (mgr.25304) 334 : cluster [DBG] pgmap v609: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1k op/s 2024-09-13T00:47:02.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:02 smithi077 bash[80681]: cluster 2024-09-13T00:47:00.586785+0000 mgr.y (mgr.25304) 334 : cluster [DBG] pgmap v609: 457 pgs: 457 active+clean; 83 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 5.8 MiB/s wr, 1k op/s 2024-09-13T00:47:04.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:04 smithi077 bash[80681]: cluster 2024-09-13T00:47:02.588433+0000 mgr.y (mgr.25304) 335 : cluster [DBG] pgmap v611: 457 pgs: 457 active+clean; 91 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.7 MiB/s wr, 781 op/s 2024-09-13T00:47:04.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:04 smithi077 bash[80681]: audit 2024-09-13T00:47:03.153799+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:04.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:04 smithi077 bash[83127]: cluster 2024-09-13T00:47:02.588433+0000 mgr.y (mgr.25304) 335 : cluster [DBG] pgmap v611: 457 pgs: 457 active+clean; 91 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.7 MiB/s wr, 781 op/s 2024-09-13T00:47:04.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:04 smithi077 bash[83127]: audit 2024-09-13T00:47:03.153799+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:04 smithi088 bash[68960]: cluster 2024-09-13T00:47:02.588433+0000 mgr.y (mgr.25304) 335 : cluster [DBG] pgmap v611: 457 pgs: 457 active+clean; 91 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.7 MiB/s wr, 781 op/s 2024-09-13T00:47:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:04 smithi088 bash[68960]: audit 2024-09-13T00:47:03.153799+0000 mon.a (mon.0) 1049 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:04.851 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:47:04.851 INFO:tasks.workunit.client.0.smithi077.stdout:finished read 2024-09-13T00:47:04.851 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 0 2024-09-13T00:47:04.851 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.851 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.851 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 0 2024-09-13T00:47:04.851 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.851 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.851 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 16384 2024-09-13T00:47:04.852 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.852 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.852 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 16384 2024-09-13T00:47:04.852 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.852 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.852 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:04.852 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:04.852 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 49152 2024-09-13T00:47:04.852 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.852 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.852 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 49152 2024-09-13T00:47:04.852 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.852 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.853 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 65536 2024-09-13T00:47:04.853 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.853 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.853 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 65536 2024-09-13T00:47:04.853 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.853 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.853 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:04.853 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:04.853 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 98304 2024-09-13T00:47:04.853 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.853 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.853 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 98304 2024-09-13T00:47:04.853 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.854 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.854 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 114688 2024-09-13T00:47:04.854 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.854 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.854 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 114688 2024-09-13T00:47:04.854 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.854 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.854 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:04.854 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:04.854 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 147456 2024-09-13T00:47:04.854 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.854 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.854 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 147456 2024-09-13T00:47:04.855 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.855 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.855 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 163840 2024-09-13T00:47:04.855 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.855 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.855 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 163840 2024-09-13T00:47:04.855 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.855 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.855 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:04.855 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:04.855 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 196608 2024-09-13T00:47:04.855 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.856 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.856 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 196608 2024-09-13T00:47:04.856 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.856 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.856 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 212992 2024-09-13T00:47:04.856 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.856 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.856 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 212992 2024-09-13T00:47:04.856 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.856 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.856 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:04.857 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:04.857 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:04.857 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:04.857 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:04.857 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:04.857 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:04.857 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.857 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.857 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:04.857 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:04.857 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:04.858 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:04.858 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:04.858 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.858 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.858 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:04.858 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:04.858 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:04.876 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:04.876 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:04.876 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.877 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.877 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:04.877 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:04.877 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:04.898 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:04.898 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:04.898 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.898 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.898 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:04.898 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:04.898 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:04.899 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:04.899 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:04.899 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:04.899 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:04.899 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:04.899 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:04.962 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:04.962 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:04.962 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:04.963 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:04.963 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:04.963 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:04.963 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:05.036 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:05.036 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:05.036 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:05.036 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:05.036 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:05.036 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:05.037 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:05.143 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:05.143 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:05.143 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:05.143 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:05.143 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:05.144 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:05.144 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:05.221 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:05.222 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:05.222 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:05.222 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:05.222 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:05.222 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:05.222 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:05.222 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:05.222 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:05.222 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:05.222 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:05.222 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:05.223 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:05.344 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:05.344 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:05.344 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:05.344 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:05.345 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:05.345 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:05.345 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:05.476 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:05.476 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:05.476 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:05.476 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:05.476 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:05.477 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:05.477 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:05.632 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:05.633 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:05.633 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:05.633 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:05.633 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:05.633 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:05.633 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:05.785 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:05.785 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:05.785 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:05.785 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:05.785 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:05.785 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:05.785 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:05.786 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:05.786 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:05.786 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:05.786 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:05.786 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:05.786 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:05.989 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:05.989 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:05.989 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:05.989 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:05.989 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:05.989 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:05.989 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:06.190 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:06.190 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:06.190 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:06.190 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:06.190 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:06.190 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:06.190 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:06.399 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:06.399 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:06.399 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:06.400 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:06.400 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:06.400 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:06.400 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:06.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:06 smithi077 bash[83127]: cluster 2024-09-13T00:47:04.591844+0000 mgr.y (mgr.25304) 336 : cluster [DBG] pgmap v612: 457 pgs: 457 active+clean; 127 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.7 MiB/s wr, 959 op/s 2024-09-13T00:47:06.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:06 smithi077 bash[80681]: cluster 2024-09-13T00:47:04.591844+0000 mgr.y (mgr.25304) 336 : cluster [DBG] pgmap v612: 457 pgs: 457 active+clean; 127 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.7 MiB/s wr, 959 op/s 2024-09-13T00:47:06.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:06 smithi088 bash[68960]: cluster 2024-09-13T00:47:04.591844+0000 mgr.y (mgr.25304) 336 : cluster [DBG] pgmap v612: 457 pgs: 457 active+clean; 127 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.7 MiB/s wr, 959 op/s 2024-09-13T00:47:06.605 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:06.605 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:06.605 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:06.606 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:06.606 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:06.606 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:06.606 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:06.606 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:06.606 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:06.606 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:06.606 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:06.606 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:06.606 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:06.889 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:06.889 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:06.889 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:06.889 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:06.889 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:06.889 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:06.889 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:07.144 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:07.145 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:07.145 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:07.145 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:07.145 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:07.145 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:07.145 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:07.417 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:07.417 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:07.417 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:07.417 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:07.417 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:07.418 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:07.418 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:47:07.696 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:47:07.696 INFO:tasks.workunit.client.0.smithi077.stdout:finished writesame 2024-09-13T00:47:07.696 INFO:tasks.workunit.client.0.smithi077.stdout:to verify the data 2024-09-13T00:47:07.696 INFO:tasks.workunit.client.0.smithi077.stdout:verified 2024-09-13T00:47:07.696 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:07.696 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:07.696 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:07.697 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:47:07.697 INFO:tasks.workunit.client.0.smithi077.stdout:started writesame 2024-09-13T00:47:07.697 INFO:tasks.workunit.client.0.smithi077.stdout:expected fail, finished writesame 2024-09-13T00:47:07.697 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestIOPPWithIOHint (5546 ms) 2024-09-13T00:47:07.697 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestIOToSnapshot 2024-09-13T00:47:07.697 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:08.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:08 smithi077 bash[83127]: cluster 2024-09-13T00:47:06.593599+0000 mgr.y (mgr.25304) 337 : cluster [DBG] pgmap v613: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 7.4 MiB/s wr, 1.74k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:47:08.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:08 smithi077 bash[80681]: cluster 2024-09-13T00:47:06.593599+0000 mgr.y (mgr.25304) 337 : cluster [DBG] pgmap v613: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 7.4 MiB/s wr, 1.74k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:47:08.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:08 smithi088 bash[68960]: cluster 2024-09-13T00:47:06.593599+0000 mgr.y (mgr.25304) 337 : cluster [DBG] pgmap v613: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 7.4 MiB/s wr, 1.74k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:47:08.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:47:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:47:08] "GET /metrics HTTP/1.1" 200 63096 "" "Prometheus/2.43.0" 2024-09-13T00:47:09.259 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 0 2024-09-13T00:47:09.259 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 0 2024-09-13T00:47:09.259 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 1 2024-09-13T00:47:09.259 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 1 2024-09-13T00:47:09.259 INFO:tasks.workunit.client.0.smithi077.stdout:snap: orig 2024-09-13T00:47:09.259 INFO:tasks.workunit.client.0.smithi077.stdout:found orig with size 2097152 2024-09-13T00:47:09.259 INFO:tasks.workunit.client.0.smithi077.stdout:read: 80 2024-09-13T00:47:09.259 INFO:tasks.workunit.client.0.smithi077.stdout:write test data! 2024-09-13T00:47:09.259 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 80 2024-09-13T00:47:09.259 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 2 2024-09-13T00:47:09.260 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 2 2024-09-13T00:47:09.260 INFO:tasks.workunit.client.0.smithi077.stdout:snap: orig 2024-09-13T00:47:09.260 INFO:tasks.workunit.client.0.smithi077.stdout:snap: written 2024-09-13T00:47:09.260 INFO:tasks.workunit.client.0.smithi077.stdout:found orig with size 2097152 2024-09-13T00:47:09.260 INFO:tasks.workunit.client.0.smithi077.stdout:found written with size 2097152 2024-09-13T00:47:09.260 INFO:tasks.workunit.client.0.smithi077.stdout:read: 80 2024-09-13T00:47:09.260 INFO:tasks.workunit.client.0.smithi077.stdout:read: 80 2024-09-13T00:47:09.260 INFO:tasks.workunit.client.0.smithi077.stdout:read: 80 2024-09-13T00:47:09.260 INFO:tasks.workunit.client.0.smithi077.stdout:write to snapshot returned -30 2024-09-13T00:47:09.260 INFO:tasks.workunit.client.0.smithi077.stdout:Read-only file system 2024-09-13T00:47:09.313 INFO:tasks.workunit.client.0.smithi077.stdout:read: 80 2024-09-13T00:47:09.313 INFO:tasks.workunit.client.0.smithi077.stdout:read: 80 2024-09-13T00:47:09.314 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 80 2024-09-13T00:47:09.314 INFO:tasks.workunit.client.0.smithi077.stdout:opening testimg@orig 2024-09-13T00:47:09.314 INFO:tasks.workunit.client.0.smithi077.stdout:read: 80 2024-09-13T00:47:09.314 INFO:tasks.workunit.client.0.smithi077.stdout:write to snapshot returned -30 2024-09-13T00:47:09.314 INFO:tasks.workunit.client.0.smithi077.stdout:Read-only file system 2024-09-13T00:47:09.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:09 smithi077 bash[80681]: cluster 2024-09-13T00:47:08.175232+0000 mon.a (mon.0) 1050 : cluster [DBG] osdmap e657: 8 total, 8 up, 8 in 2024-09-13T00:47:09.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:09 smithi077 bash[83127]: cluster 2024-09-13T00:47:08.175232+0000 mon.a (mon.0) 1050 : cluster [DBG] osdmap e657: 8 total, 8 up, 8 in 2024-09-13T00:47:09.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:09 smithi088 bash[68960]: cluster 2024-09-13T00:47:08.175232+0000 mon.a (mon.0) 1050 : cluster [DBG] osdmap e657: 8 total, 8 up, 8 in 2024-09-13T00:47:10.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:10 smithi077 bash[80681]: cluster 2024-09-13T00:47:08.595025+0000 mgr.y (mgr.25304) 338 : cluster [DBG] pgmap v615: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 7.2 MiB/s wr, 994 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:47:10.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:10 smithi077 bash[80681]: cluster 2024-09-13T00:47:09.184482+0000 mon.a (mon.0) 1051 : cluster [DBG] osdmap e658: 8 total, 8 up, 8 in 2024-09-13T00:47:10.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:10 smithi077 bash[83127]: cluster 2024-09-13T00:47:08.595025+0000 mgr.y (mgr.25304) 338 : cluster [DBG] pgmap v615: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 7.2 MiB/s wr, 994 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:47:10.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:10 smithi077 bash[83127]: cluster 2024-09-13T00:47:09.184482+0000 mon.a (mon.0) 1051 : cluster [DBG] osdmap e658: 8 total, 8 up, 8 in 2024-09-13T00:47:10.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:10 smithi088 bash[68960]: cluster 2024-09-13T00:47:08.595025+0000 mgr.y (mgr.25304) 338 : cluster [DBG] pgmap v615: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 7.2 MiB/s wr, 994 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:47:10.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:10 smithi088 bash[68960]: cluster 2024-09-13T00:47:09.184482+0000 mon.a (mon.0) 1051 : cluster [DBG] osdmap e658: 8 total, 8 up, 8 in 2024-09-13T00:47:11.265 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 2 2024-09-13T00:47:11.265 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 2 2024-09-13T00:47:11.266 INFO:tasks.workunit.client.0.smithi077.stdout:snap: orig 2024-09-13T00:47:11.266 INFO:tasks.workunit.client.0.smithi077.stdout:snap: written 2024-09-13T00:47:11.266 INFO:tasks.workunit.client.0.smithi077.stdout:found orig with size 2097152 2024-09-13T00:47:11.266 INFO:tasks.workunit.client.0.smithi077.stdout:found written with size 2097152 2024-09-13T00:47:11.266 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 1 2024-09-13T00:47:11.266 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 1 2024-09-13T00:47:11.266 INFO:tasks.workunit.client.0.smithi077.stdout:snap: orig 2024-09-13T00:47:11.266 INFO:tasks.workunit.client.0.smithi077.stdout:found orig with size 2097152 2024-09-13T00:47:11.266 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 0 2024-09-13T00:47:11.266 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 0 2024-09-13T00:47:11.266 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestIOToSnapshot (3570 ms) 2024-09-13T00:47:11.266 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestSnapshotDeletedIo 2024-09-13T00:47:11.266 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:11.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:11 smithi077 bash[80681]: cluster 2024-09-13T00:47:10.200377+0000 mon.a (mon.0) 1052 : cluster [DBG] osdmap e659: 8 total, 8 up, 8 in 2024-09-13T00:47:11.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:11 smithi077 bash[83127]: cluster 2024-09-13T00:47:10.200377+0000 mon.a (mon.0) 1052 : cluster [DBG] osdmap e659: 8 total, 8 up, 8 in 2024-09-13T00:47:11.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:11 smithi088 bash[68960]: cluster 2024-09-13T00:47:10.200377+0000 mon.a (mon.0) 1052 : cluster [DBG] osdmap e659: 8 total, 8 up, 8 in 2024-09-13T00:47:12.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:12 smithi077 bash[80681]: cluster 2024-09-13T00:47:10.596123+0000 mgr.y (mgr.25304) 339 : cluster [DBG] pgmap v618: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 939 KiB/s wr, 1.07k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:47:12.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:12 smithi077 bash[80681]: cluster 2024-09-13T00:47:11.210960+0000 mon.a (mon.0) 1053 : cluster [DBG] osdmap e660: 8 total, 8 up, 8 in 2024-09-13T00:47:12.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:12 smithi077 bash[83127]: cluster 2024-09-13T00:47:10.596123+0000 mgr.y (mgr.25304) 339 : cluster [DBG] pgmap v618: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 939 KiB/s wr, 1.07k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:47:12.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:12 smithi077 bash[83127]: cluster 2024-09-13T00:47:11.210960+0000 mon.a (mon.0) 1053 : cluster [DBG] osdmap e660: 8 total, 8 up, 8 in 2024-09-13T00:47:12.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:12 smithi088 bash[68960]: cluster 2024-09-13T00:47:10.596123+0000 mgr.y (mgr.25304) 339 : cluster [DBG] pgmap v618: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 939 KiB/s wr, 1.07k op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:47:12.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:12 smithi088 bash[68960]: cluster 2024-09-13T00:47:11.210960+0000 mon.a (mon.0) 1053 : cluster [DBG] osdmap e660: 8 total, 8 up, 8 in 2024-09-13T00:47:12.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:47:12 smithi088 bash[61791]: ts=2024-09-13T00:47:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:47:13.277 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestSnapshotDeletedIo (2012 ms) 2024-09-13T00:47:13.278 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestClone 2024-09-13T00:47:13.278 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:13.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:13 smithi077 bash[83127]: cluster 2024-09-13T00:47:12.212143+0000 mon.a (mon.0) 1054 : cluster [DBG] osdmap e661: 8 total, 8 up, 8 in 2024-09-13T00:47:13.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:13 smithi077 bash[80681]: cluster 2024-09-13T00:47:12.212143+0000 mon.a (mon.0) 1054 : cluster [DBG] osdmap e661: 8 total, 8 up, 8 in 2024-09-13T00:47:13.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:13 smithi088 bash[68960]: cluster 2024-09-13T00:47:12.212143+0000 mon.a (mon.0) 1054 : cluster [DBG] osdmap e661: 8 total, 8 up, 8 in 2024-09-13T00:47:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:14 smithi077 bash[80681]: cluster 2024-09-13T00:47:12.597134+0000 mgr.y (mgr.25304) 340 : cluster [DBG] pgmap v621: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 325 KiB/s rd, 145 KiB/s wr, 626 op/s 2024-09-13T00:47:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:14 smithi077 bash[80681]: cluster 2024-09-13T00:47:13.235020+0000 mon.a (mon.0) 1055 : cluster [DBG] osdmap e662: 8 total, 8 up, 8 in 2024-09-13T00:47:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:14 smithi077 bash[80681]: audit 2024-09-13T00:47:13.421675+0000 mon.a (mon.0) 1056 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:14 smithi077 bash[83127]: cluster 2024-09-13T00:47:12.597134+0000 mgr.y (mgr.25304) 340 : cluster [DBG] pgmap v621: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 325 KiB/s rd, 145 KiB/s wr, 626 op/s 2024-09-13T00:47:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:14 smithi077 bash[83127]: cluster 2024-09-13T00:47:13.235020+0000 mon.a (mon.0) 1055 : cluster [DBG] osdmap e662: 8 total, 8 up, 8 in 2024-09-13T00:47:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:14 smithi077 bash[83127]: audit 2024-09-13T00:47:13.421675+0000 mon.a (mon.0) 1056 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:14 smithi088 bash[68960]: cluster 2024-09-13T00:47:12.597134+0000 mgr.y (mgr.25304) 340 : cluster [DBG] pgmap v621: 457 pgs: 457 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 325 KiB/s rd, 145 KiB/s wr, 626 op/s 2024-09-13T00:47:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:14 smithi088 bash[68960]: cluster 2024-09-13T00:47:13.235020+0000 mon.a (mon.0) 1055 : cluster [DBG] osdmap e662: 8 total, 8 up, 8 in 2024-09-13T00:47:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:14 smithi088 bash[68960]: audit 2024-09-13T00:47:13.421675+0000 mon.a (mon.0) 1056 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:15.293 INFO:tasks.workunit.client.0.smithi077.stdout:made parent image "parent" 2024-09-13T00:47:15.294 INFO:tasks.workunit.client.0.smithi077.stdout:parent has no parent info 2024-09-13T00:47:15.294 INFO:tasks.workunit.client.0.smithi077.stdout:made snapshot "parent@parent_snap" 2024-09-13T00:47:15.294 INFO:tasks.workunit.client.0.smithi077.stdout:can't unprotect an unprotected snap 2024-09-13T00:47:15.294 INFO:tasks.workunit.client.0.smithi077.stdout:can't protect a protected snap 2024-09-13T00:47:15.294 INFO:tasks.workunit.client.0.smithi077.stdout:made and opened clone "child" 2024-09-13T00:47:15.294 INFO:tasks.workunit.client.0.smithi077.stdout:read: 8 2024-09-13T00:47:15.294 INFO:tasks.workunit.client.0.smithi077.stdout:read: 8 2024-09-13T00:47:15.294 INFO:tasks.workunit.client.0.smithi077.stdout:read: 8 2024-09-13T00:47:15.294 INFO:tasks.workunit.client.0.smithi077.stdout:sizes and overlaps are good between parent and child 2024-09-13T00:47:15.294 INFO:tasks.workunit.client.0.smithi077.stdout:child image successfully cloned all image-meta pairs 2024-09-13T00:47:15.294 INFO:tasks.workunit.client.0.smithi077.stdout:sized down clone, changed overlap 2024-09-13T00:47:15.295 INFO:tasks.workunit.client.0.smithi077.stdout:parent info: size 4194304 obj_size 4194304 parent_pool 18446744073709551615 2024-09-13T00:47:15.295 INFO:tasks.workunit.client.0.smithi077.stdout:sized up clone, changed size but not overlap or parent's size 2024-09-13T00:47:15.295 INFO:tasks.workunit.client.0.smithi077.stdout:can't remove parent while child still exists 2024-09-13T00:47:15.295 INFO:tasks.workunit.client.0.smithi077.stdout:can't remove parent while still protected 2024-09-13T00:47:15.295 INFO:tasks.workunit.client.0.smithi077.stdout:removed parent snap after unprotecting 2024-09-13T00:47:15.295 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestClone (2016 ms) 2024-09-13T00:47:15.295 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestClone2 2024-09-13T00:47:15.295 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:15.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:15 smithi077 bash[83127]: cluster 2024-09-13T00:47:14.245494+0000 mon.a (mon.0) 1057 : cluster [DBG] osdmap e663: 8 total, 8 up, 8 in 2024-09-13T00:47:15.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:15 smithi077 bash[80681]: cluster 2024-09-13T00:47:14.245494+0000 mon.a (mon.0) 1057 : cluster [DBG] osdmap e663: 8 total, 8 up, 8 in 2024-09-13T00:47:15.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:15 smithi088 bash[68960]: cluster 2024-09-13T00:47:14.245494+0000 mon.a (mon.0) 1057 : cluster [DBG] osdmap e663: 8 total, 8 up, 8 in 2024-09-13T00:47:16.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:16 smithi077 bash[80681]: cluster 2024-09-13T00:47:14.599433+0000 mgr.y (mgr.25304) 341 : cluster [DBG] pgmap v624: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 307 KiB/s rd, 146 KiB/s wr, 607 op/s 2024-09-13T00:47:16.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:16 smithi077 bash[80681]: cluster 2024-09-13T00:47:15.260414+0000 mon.a (mon.0) 1058 : cluster [DBG] osdmap e664: 8 total, 8 up, 8 in 2024-09-13T00:47:16.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:16 smithi077 bash[83127]: cluster 2024-09-13T00:47:14.599433+0000 mgr.y (mgr.25304) 341 : cluster [DBG] pgmap v624: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 307 KiB/s rd, 146 KiB/s wr, 607 op/s 2024-09-13T00:47:16.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:16 smithi077 bash[83127]: cluster 2024-09-13T00:47:15.260414+0000 mon.a (mon.0) 1058 : cluster [DBG] osdmap e664: 8 total, 8 up, 8 in 2024-09-13T00:47:16.568 INFO:tasks.workunit.client.0.smithi077.stdout:made parent image "parent" 2024-09-13T00:47:16.568 INFO:tasks.workunit.client.0.smithi077.stdout:parent has no parent info 2024-09-13T00:47:16.568 INFO:tasks.workunit.client.0.smithi077.stdout:made snapshot "parent@parent_snap" 2024-09-13T00:47:16.569 INFO:tasks.workunit.client.0.smithi077.stdout:made and opened clone "child" 2024-09-13T00:47:16.569 INFO:tasks.workunit.client.0.smithi077.stdout:child image successfully cloned all image-meta pairs 2024-09-13T00:47:16.569 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestClone2 (1275 ms) 2024-09-13T00:47:16.569 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ListChildren 2024-09-13T00:47:16.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:16 smithi088 bash[68960]: cluster 2024-09-13T00:47:14.599433+0000 mgr.y (mgr.25304) 341 : cluster [DBG] pgmap v624: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 307 KiB/s rd, 146 KiB/s wr, 607 op/s 2024-09-13T00:47:16.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:16 smithi088 bash[68960]: cluster 2024-09-13T00:47:15.260414+0000 mon.a (mon.0) 1058 : cluster [DBG] osdmap e664: 8 total, 8 up, 8 in 2024-09-13T00:47:17.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:17 smithi077 bash[80681]: cluster 2024-09-13T00:47:16.259674+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e665: 8 total, 8 up, 8 in 2024-09-13T00:47:17.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:17 smithi077 bash[83127]: cluster 2024-09-13T00:47:16.259674+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e665: 8 total, 8 up, 8 in 2024-09-13T00:47:17.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:17 smithi088 bash[68960]: cluster 2024-09-13T00:47:16.259674+0000 mon.a (mon.0) 1059 : cluster [DBG] osdmap e665: 8 total, 8 up, 8 in 2024-09-13T00:47:18.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:18 smithi077 bash[83127]: cluster 2024-09-13T00:47:16.600448+0000 mgr.y (mgr.25304) 342 : cluster [DBG] pgmap v627: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 344 KiB/s rd, 25 KiB/s wr, 418 op/s 2024-09-13T00:47:18.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:18 smithi077 bash[83127]: cluster 2024-09-13T00:47:17.283712+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e666: 8 total, 8 up, 8 in 2024-09-13T00:47:18.525 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:18 smithi077 bash[83127]: audit 2024-09-13T00:47:17.289014+0000 mon.a (mon.0) 1061 : audit [INF] from='client.? 172.21.15.77:0/2771553826' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:18.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:18 smithi077 bash[80681]: cluster 2024-09-13T00:47:16.600448+0000 mgr.y (mgr.25304) 342 : cluster [DBG] pgmap v627: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 344 KiB/s rd, 25 KiB/s wr, 418 op/s 2024-09-13T00:47:18.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:18 smithi077 bash[80681]: cluster 2024-09-13T00:47:17.283712+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e666: 8 total, 8 up, 8 in 2024-09-13T00:47:18.525 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:18 smithi077 bash[80681]: audit 2024-09-13T00:47:17.289014+0000 mon.a (mon.0) 1061 : audit [INF] from='client.? 172.21.15.77:0/2771553826' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:18.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:18 smithi088 bash[68960]: cluster 2024-09-13T00:47:16.600448+0000 mgr.y (mgr.25304) 342 : cluster [DBG] pgmap v627: 457 pgs: 457 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 344 KiB/s rd, 25 KiB/s wr, 418 op/s 2024-09-13T00:47:18.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:18 smithi088 bash[68960]: cluster 2024-09-13T00:47:17.283712+0000 mon.a (mon.0) 1060 : cluster [DBG] osdmap e666: 8 total, 8 up, 8 in 2024-09-13T00:47:18.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:18 smithi088 bash[68960]: audit 2024-09-13T00:47:17.289014+0000 mon.a (mon.0) 1061 : audit [INF] from='client.? 172.21.15.77:0/2771553826' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:18.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:47:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:47:18] "GET /metrics HTTP/1.1" 200 63098 "" "Prometheus/2.43.0" 2024-09-13T00:47:19.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:19 smithi088 bash[68960]: audit 2024-09-13T00:47:18.294466+0000 mon.a (mon.0) 1062 : audit [INF] from='client.? 172.21.15.77:0/2771553826' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:47:19.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:19 smithi088 bash[68960]: cluster 2024-09-13T00:47:18.294996+0000 mon.a (mon.0) 1063 : cluster [DBG] osdmap e667: 8 total, 8 up, 8 in 2024-09-13T00:47:19.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:19 smithi077 bash[80681]: audit 2024-09-13T00:47:18.294466+0000 mon.a (mon.0) 1062 : audit [INF] from='client.? 172.21.15.77:0/2771553826' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:47:19.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:19 smithi077 bash[80681]: cluster 2024-09-13T00:47:18.294996+0000 mon.a (mon.0) 1063 : cluster [DBG] osdmap e667: 8 total, 8 up, 8 in 2024-09-13T00:47:19.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:19 smithi077 bash[83127]: audit 2024-09-13T00:47:18.294466+0000 mon.a (mon.0) 1062 : audit [INF] from='client.? 172.21.15.77:0/2771553826' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:47:19.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:19 smithi077 bash[83127]: cluster 2024-09-13T00:47:18.294996+0000 mon.a (mon.0) 1063 : cluster [DBG] osdmap e667: 8 total, 8 up, 8 in 2024-09-13T00:47:20.336 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:20.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:20 smithi088 bash[68960]: cluster 2024-09-13T00:47:18.601991+0000 mgr.y (mgr.25304) 343 : cluster [DBG] pgmap v630: 489 pgs: 10 creating+peering, 21 unknown, 458 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 370 KiB/s rd, 23 KiB/s wr, 437 op/s 2024-09-13T00:47:20.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:20 smithi088 bash[68960]: cluster 2024-09-13T00:47:19.311397+0000 mon.a (mon.0) 1064 : cluster [DBG] osdmap e668: 8 total, 8 up, 8 in 2024-09-13T00:47:20.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:20 smithi088 bash[68960]: audit 2024-09-13T00:47:19.313267+0000 mon.c (mon.2) 46 : audit [INF] from='client.? 172.21.15.77:0/1566866251' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:20.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:20 smithi088 bash[68960]: audit 2024-09-13T00:47:19.323676+0000 mon.a (mon.0) 1065 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:20.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:20 smithi077 bash[80681]: cluster 2024-09-13T00:47:18.601991+0000 mgr.y (mgr.25304) 343 : cluster [DBG] pgmap v630: 489 pgs: 10 creating+peering, 21 unknown, 458 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 370 KiB/s rd, 23 KiB/s wr, 437 op/s 2024-09-13T00:47:20.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:20 smithi077 bash[80681]: cluster 2024-09-13T00:47:19.311397+0000 mon.a (mon.0) 1064 : cluster [DBG] osdmap e668: 8 total, 8 up, 8 in 2024-09-13T00:47:20.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:20 smithi077 bash[80681]: audit 2024-09-13T00:47:19.313267+0000 mon.c (mon.2) 46 : audit [INF] from='client.? 172.21.15.77:0/1566866251' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:20.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:20 smithi077 bash[80681]: audit 2024-09-13T00:47:19.323676+0000 mon.a (mon.0) 1065 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:20.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:20 smithi077 bash[83127]: cluster 2024-09-13T00:47:18.601991+0000 mgr.y (mgr.25304) 343 : cluster [DBG] pgmap v630: 489 pgs: 10 creating+peering, 21 unknown, 458 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 370 KiB/s rd, 23 KiB/s wr, 437 op/s 2024-09-13T00:47:20.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:20 smithi077 bash[83127]: cluster 2024-09-13T00:47:19.311397+0000 mon.a (mon.0) 1064 : cluster [DBG] osdmap e668: 8 total, 8 up, 8 in 2024-09-13T00:47:20.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:20 smithi077 bash[83127]: audit 2024-09-13T00:47:19.313267+0000 mon.c (mon.2) 46 : audit [INF] from='client.? 172.21.15.77:0/1566866251' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:20.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:20 smithi077 bash[83127]: audit 2024-09-13T00:47:19.323676+0000 mon.a (mon.0) 1065 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:21.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:21 smithi088 bash[68960]: audit 2024-09-13T00:47:20.309026+0000 mon.a (mon.0) 1066 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:47:21.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:21 smithi088 bash[68960]: cluster 2024-09-13T00:47:20.309486+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e669: 8 total, 8 up, 8 in 2024-09-13T00:47:21.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:21 smithi088 bash[68960]: cluster 2024-09-13T00:47:20.997721+0000 mon.a (mon.0) 1068 : cluster [DBG] osdmap e670: 8 total, 8 up, 8 in 2024-09-13T00:47:21.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:21 smithi077 bash[80681]: audit 2024-09-13T00:47:20.309026+0000 mon.a (mon.0) 1066 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:47:21.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:21 smithi077 bash[80681]: cluster 2024-09-13T00:47:20.309486+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e669: 8 total, 8 up, 8 in 2024-09-13T00:47:21.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:21 smithi077 bash[80681]: cluster 2024-09-13T00:47:20.997721+0000 mon.a (mon.0) 1068 : cluster [DBG] osdmap e670: 8 total, 8 up, 8 in 2024-09-13T00:47:21.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:21 smithi077 bash[83127]: audit 2024-09-13T00:47:20.309026+0000 mon.a (mon.0) 1066 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:47:21.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:21 smithi077 bash[83127]: cluster 2024-09-13T00:47:20.309486+0000 mon.a (mon.0) 1067 : cluster [DBG] osdmap e669: 8 total, 8 up, 8 in 2024-09-13T00:47:21.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:21 smithi077 bash[83127]: cluster 2024-09-13T00:47:20.997721+0000 mon.a (mon.0) 1068 : cluster [DBG] osdmap e670: 8 total, 8 up, 8 in 2024-09-13T00:47:22.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:22 smithi088 bash[68960]: cluster 2024-09-13T00:47:20.603129+0000 mgr.y (mgr.25304) 344 : cluster [DBG] pgmap v633: 521 pgs: 21 creating+peering, 25 unknown, 475 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 463 KiB/s rd, 34 KiB/s wr, 553 op/s 2024-09-13T00:47:22.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:22 smithi088 bash[68960]: cluster 2024-09-13T00:47:22.001854+0000 mon.a (mon.0) 1069 : cluster [DBG] osdmap e671: 8 total, 8 up, 8 in 2024-09-13T00:47:22.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:47:22 smithi088 bash[61791]: ts=2024-09-13T00:47:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:47:22.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:22 smithi077 bash[80681]: cluster 2024-09-13T00:47:20.603129+0000 mgr.y (mgr.25304) 344 : cluster [DBG] pgmap v633: 521 pgs: 21 creating+peering, 25 unknown, 475 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 463 KiB/s rd, 34 KiB/s wr, 553 op/s 2024-09-13T00:47:22.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:22 smithi077 bash[80681]: cluster 2024-09-13T00:47:22.001854+0000 mon.a (mon.0) 1069 : cluster [DBG] osdmap e671: 8 total, 8 up, 8 in 2024-09-13T00:47:22.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:22 smithi077 bash[83127]: cluster 2024-09-13T00:47:20.603129+0000 mgr.y (mgr.25304) 344 : cluster [DBG] pgmap v633: 521 pgs: 21 creating+peering, 25 unknown, 475 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 463 KiB/s rd, 34 KiB/s wr, 553 op/s 2024-09-13T00:47:22.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:22 smithi077 bash[83127]: cluster 2024-09-13T00:47:22.001854+0000 mon.a (mon.0) 1069 : cluster [DBG] osdmap e671: 8 total, 8 up, 8 in 2024-09-13T00:47:24.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:23 smithi077 bash[80681]: cluster 2024-09-13T00:47:22.604189+0000 mgr.y (mgr.25304) 345 : cluster [DBG] pgmap v636: 521 pgs: 21 creating+peering, 15 unknown, 485 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 417 KiB/s rd, 32 KiB/s wr, 502 op/s 2024-09-13T00:47:24.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:23 smithi077 bash[80681]: cluster 2024-09-13T00:47:23.005495+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2024-09-13T00:47:24.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:24 smithi077 bash[83127]: cluster 2024-09-13T00:47:22.604189+0000 mgr.y (mgr.25304) 345 : cluster [DBG] pgmap v636: 521 pgs: 21 creating+peering, 15 unknown, 485 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 417 KiB/s rd, 32 KiB/s wr, 502 op/s 2024-09-13T00:47:24.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:24 smithi077 bash[83127]: cluster 2024-09-13T00:47:23.005495+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2024-09-13T00:47:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:24 smithi088 bash[68960]: cluster 2024-09-13T00:47:22.604189+0000 mgr.y (mgr.25304) 345 : cluster [DBG] pgmap v636: 521 pgs: 21 creating+peering, 15 unknown, 485 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 417 KiB/s rd, 32 KiB/s wr, 502 op/s 2024-09-13T00:47:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:24 smithi088 bash[68960]: cluster 2024-09-13T00:47:23.005495+0000 mon.a (mon.0) 1070 : cluster [DBG] osdmap e672: 8 total, 8 up, 8 in 2024-09-13T00:47:25.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:25 smithi077 bash[83127]: cluster 2024-09-13T00:47:24.020039+0000 mon.a (mon.0) 1071 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2024-09-13T00:47:25.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:25 smithi077 bash[80681]: cluster 2024-09-13T00:47:24.020039+0000 mon.a (mon.0) 1071 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2024-09-13T00:47:25.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:25 smithi088 bash[68960]: cluster 2024-09-13T00:47:24.020039+0000 mon.a (mon.0) 1071 : cluster [DBG] osdmap e673: 8 total, 8 up, 8 in 2024-09-13T00:47:26.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:26 smithi077 bash[80681]: cluster 2024-09-13T00:47:24.607208+0000 mgr.y (mgr.25304) 346 : cluster [DBG] pgmap v639: 521 pgs: 2 creating+peering, 519 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 1.5 KiB/s wr, 13 op/s 2024-09-13T00:47:26.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:26 smithi077 bash[80681]: cluster 2024-09-13T00:47:25.029002+0000 mon.a (mon.0) 1072 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2024-09-13T00:47:26.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:26 smithi077 bash[80681]: cluster 2024-09-13T00:47:26.001283+0000 mon.a (mon.0) 1073 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2024-09-13T00:47:26.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:26 smithi077 bash[83127]: cluster 2024-09-13T00:47:24.607208+0000 mgr.y (mgr.25304) 346 : cluster [DBG] pgmap v639: 521 pgs: 2 creating+peering, 519 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 1.5 KiB/s wr, 13 op/s 2024-09-13T00:47:26.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:26 smithi077 bash[83127]: cluster 2024-09-13T00:47:25.029002+0000 mon.a (mon.0) 1072 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2024-09-13T00:47:26.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:26 smithi077 bash[83127]: cluster 2024-09-13T00:47:26.001283+0000 mon.a (mon.0) 1073 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2024-09-13T00:47:26.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:26 smithi088 bash[68960]: cluster 2024-09-13T00:47:24.607208+0000 mgr.y (mgr.25304) 346 : cluster [DBG] pgmap v639: 521 pgs: 2 creating+peering, 519 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.7 KiB/s rd, 1.5 KiB/s wr, 13 op/s 2024-09-13T00:47:26.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:26 smithi088 bash[68960]: cluster 2024-09-13T00:47:25.029002+0000 mon.a (mon.0) 1072 : cluster [DBG] osdmap e674: 8 total, 8 up, 8 in 2024-09-13T00:47:26.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:26 smithi088 bash[68960]: cluster 2024-09-13T00:47:26.001283+0000 mon.a (mon.0) 1073 : cluster [DBG] osdmap e675: 8 total, 8 up, 8 in 2024-09-13T00:47:26.402 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.402 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.402 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.402 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.402 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.402 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 1 2024-09-13T00:47:26.402 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 1 2024-09-13T00:47:26.402 INFO:tasks.workunit.client.0.smithi077.stdout:child: image96 2024-09-13T00:47:26.402 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image96/85fe2752b736 2024-09-13T00:47:26.403 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.403 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.403 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.403 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.403 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.403 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.403 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.403 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.403 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.403 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.403 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.403 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.404 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 2 2024-09-13T00:47:26.404 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 2 2024-09-13T00:47:26.404 INFO:tasks.workunit.client.0.smithi077.stdout:child: image97 2024-09-13T00:47:26.404 INFO:tasks.workunit.client.0.smithi077.stdout:child: image96 2024-09-13T00:47:26.404 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image96/85fe2752b736 2024-09-13T00:47:26.404 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.404 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-26/image97/85fe1ce633f0 2024-09-13T00:47:26.404 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.404 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.404 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.405 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.405 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.405 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.405 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.405 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.405 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.405 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.405 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.405 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.405 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.405 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-27/image98 2024-09-13T00:47:26.405 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.406 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.406 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image98 2024-09-13T00:47:26.406 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image98 2024-09-13T00:47:26.406 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.406 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 3 2024-09-13T00:47:26.406 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 3 2024-09-13T00:47:26.406 INFO:tasks.workunit.client.0.smithi077.stdout:child: image97 2024-09-13T00:47:26.406 INFO:tasks.workunit.client.0.smithi077.stdout:child: image96 2024-09-13T00:47:26.406 INFO:tasks.workunit.client.0.smithi077.stdout:child: image98 2024-09-13T00:47:26.406 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image96/85fe2752b736 2024-09-13T00:47:26.406 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.406 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-26/image97/85fe1ce633f0 2024-09-13T00:47:26.406 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.407 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image98/85fe54e84f5e 2024-09-13T00:47:26.407 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.407 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.407 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.407 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.407 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.407 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.407 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.407 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.407 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.407 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.407 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.407 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.408 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 3 2024-09-13T00:47:26.408 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 3 2024-09-13T00:47:26.408 INFO:tasks.workunit.client.0.smithi077.stdout:child: image97 2024-09-13T00:47:26.408 INFO:tasks.workunit.client.0.smithi077.stdout:child: image96 2024-09-13T00:47:26.408 INFO:tasks.workunit.client.0.smithi077.stdout:child: image98 2024-09-13T00:47:26.408 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image96/85fe2752b736 2024-09-13T00:47:26.408 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.408 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-26/image97/85fe1ce633f0 2024-09-13T00:47:26.408 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.408 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image98/85fe54e84f5e 2024-09-13T00:47:26.408 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.408 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.409 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.409 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.409 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.409 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.409 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.409 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.409 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.409 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.409 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.409 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.409 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.409 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-27/image99 2024-09-13T00:47:26.409 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.410 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.410 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image99 2024-09-13T00:47:26.410 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image99 2024-09-13T00:47:26.410 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.410 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 4 2024-09-13T00:47:26.410 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 4 2024-09-13T00:47:26.410 INFO:tasks.workunit.client.0.smithi077.stdout:child: image97 2024-09-13T00:47:26.410 INFO:tasks.workunit.client.0.smithi077.stdout:child: image96 2024-09-13T00:47:26.410 INFO:tasks.workunit.client.0.smithi077.stdout:child: image98 2024-09-13T00:47:26.410 INFO:tasks.workunit.client.0.smithi077.stdout:child: image99 2024-09-13T00:47:26.410 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image96/85fe2752b736 2024-09-13T00:47:26.410 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.410 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-26/image97/85fe1ce633f0 2024-09-13T00:47:26.410 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.411 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image98/85fe54e84f5e 2024-09-13T00:47:26.411 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.411 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image99/85fefc734587 2024-09-13T00:47:26.411 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.411 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.411 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.411 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.411 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.411 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.411 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.411 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.411 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.411 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.412 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.412 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.412 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.412 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-27/image98 2024-09-13T00:47:26.412 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.412 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.412 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image98 2024-09-13T00:47:26.412 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image98 2024-09-13T00:47:26.412 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.412 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:26.412 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-27/image99 2024-09-13T00:47:26.412 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:26.413 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image96 2024-09-13T00:47:26.413 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image98 2024-09-13T00:47:28.245 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image99 2024-09-13T00:47:28.245 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image99 2024-09-13T00:47:28.245 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.245 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 4 2024-09-13T00:47:28.245 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 4 2024-09-13T00:47:28.245 INFO:tasks.workunit.client.0.smithi077.stdout:child: image97 2024-09-13T00:47:28.245 INFO:tasks.workunit.client.0.smithi077.stdout:child: image96 2024-09-13T00:47:28.245 INFO:tasks.workunit.client.0.smithi077.stdout:child: image98 2024-09-13T00:47:28.245 INFO:tasks.workunit.client.0.smithi077.stdout:child: image99 2024-09-13T00:47:28.245 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image96/85fe2752b736 2024-09-13T00:47:28.246 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.246 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-26/image97/85fe1ce633f0 2024-09-13T00:47:28.246 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.246 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image98/85fe54e84f5e 2024-09-13T00:47:28.246 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.246 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image99/85fefc734587 2024-09-13T00:47:28.246 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.246 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.246 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:28.246 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:28.246 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:28.246 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.246 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.247 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-27/image98 2024-09-13T00:47:28.247 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:28.247 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image98 2024-09-13T00:47:28.247 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image98 2024-09-13T00:47:28.247 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.247 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.247 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-27/image99 2024-09-13T00:47:28.247 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:28.247 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image98 2024-09-13T00:47:28.247 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image99 2024-09-13T00:47:28.247 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image99 2024-09-13T00:47:28.247 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.247 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 3 2024-09-13T00:47:28.247 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 3 2024-09-13T00:47:28.248 INFO:tasks.workunit.client.0.smithi077.stdout:child: image97 2024-09-13T00:47:28.248 INFO:tasks.workunit.client.0.smithi077.stdout:child: image98 2024-09-13T00:47:28.248 INFO:tasks.workunit.client.0.smithi077.stdout:child: image99 2024-09-13T00:47:28.248 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-26/image97/85fe1ce633f0 2024-09-13T00:47:28.248 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.248 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image98/85fe54e84f5e 2024-09-13T00:47:28.248 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.248 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image99/85fefc734587 2024-09-13T00:47:28.248 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.248 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.248 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:28.248 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:28.248 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:28.249 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.249 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.249 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-27/image99 2024-09-13T00:47:28.249 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:28.249 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-27/image99 2024-09-13T00:47:28.249 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image99 2024-09-13T00:47:28.249 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.249 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 2 2024-09-13T00:47:28.249 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 2 2024-09-13T00:47:28.249 INFO:tasks.workunit.client.0.smithi077.stdout:child: image97 2024-09-13T00:47:28.249 INFO:tasks.workunit.client.0.smithi077.stdout:child: image99 2024-09-13T00:47:28.249 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-26/image97/85fe1ce633f0 2024-09-13T00:47:28.249 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.250 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-27/image99/85fefc734587 2024-09-13T00:47:28.250 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.251 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.251 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:28.251 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:28.251 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-26/image97 2024-09-13T00:47:28.251 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.251 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 1 2024-09-13T00:47:28.251 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 1 2024-09-13T00:47:28.251 INFO:tasks.workunit.client.0.smithi077.stdout:child: image97 2024-09-13T00:47:28.251 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-26/image97/85fe1ce633f0 2024-09-13T00:47:28.251 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:28.251 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 0 2024-09-13T00:47:28.251 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 0 2024-09-13T00:47:28.251 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ListChildren (11676 ms) 2024-09-13T00:47:28.252 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ListChildrenTiered 2024-09-13T00:47:28.277 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:28 smithi077 bash[80681]: cluster 2024-09-13T00:47:26.608763+0000 mgr.y (mgr.25304) 347 : cluster [DBG] pgmap v642: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 3.5 KiB/s wr, 34 op/s 2024-09-13T00:47:28.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:28 smithi077 bash[83127]: cluster 2024-09-13T00:47:26.608763+0000 mgr.y (mgr.25304) 347 : cluster [DBG] pgmap v642: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 3.5 KiB/s wr, 34 op/s 2024-09-13T00:47:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:28 smithi088 bash[68960]: cluster 2024-09-13T00:47:26.608763+0000 mgr.y (mgr.25304) 347 : cluster [DBG] pgmap v642: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 3.5 KiB/s wr, 34 op/s 2024-09-13T00:47:28.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:47:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:47:28] "GET /metrics HTTP/1.1" 200 63098 "" "Prometheus/2.43.0" 2024-09-13T00:47:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:29 smithi088 bash[68960]: cluster 2024-09-13T00:47:28.038994+0000 mon.a (mon.0) 1074 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2024-09-13T00:47:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:29 smithi088 bash[68960]: audit 2024-09-13T00:47:28.725394+0000 mon.a (mon.0) 1075 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:29.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:29 smithi077 bash[80681]: cluster 2024-09-13T00:47:28.038994+0000 mon.a (mon.0) 1074 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2024-09-13T00:47:29.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:29 smithi077 bash[80681]: audit 2024-09-13T00:47:28.725394+0000 mon.a (mon.0) 1075 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:29.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:29 smithi077 bash[83127]: cluster 2024-09-13T00:47:28.038994+0000 mon.a (mon.0) 1074 : cluster [DBG] osdmap e676: 8 total, 8 up, 8 in 2024-09-13T00:47:29.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:29 smithi077 bash[83127]: audit 2024-09-13T00:47:28.725394+0000 mon.a (mon.0) 1075 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:30.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:30 smithi088 bash[68960]: cluster 2024-09-13T00:47:28.610260+0000 mgr.y (mgr.25304) 348 : cluster [DBG] pgmap v644: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 211 KiB/s rd, 9.4 KiB/s wr, 258 op/s 2024-09-13T00:47:30.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:30 smithi088 bash[68960]: cluster 2024-09-13T00:47:29.056216+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2024-09-13T00:47:30.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:30 smithi088 bash[68960]: audit 2024-09-13T00:47:29.075699+0000 mon.a (mon.0) 1077 : audit [INF] from='client.? 172.21.15.77:0/756362502' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:30.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:30 smithi077 bash[83127]: cluster 2024-09-13T00:47:28.610260+0000 mgr.y (mgr.25304) 348 : cluster [DBG] pgmap v644: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 211 KiB/s rd, 9.4 KiB/s wr, 258 op/s 2024-09-13T00:47:30.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:30 smithi077 bash[83127]: cluster 2024-09-13T00:47:29.056216+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2024-09-13T00:47:30.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:30 smithi077 bash[83127]: audit 2024-09-13T00:47:29.075699+0000 mon.a (mon.0) 1077 : audit [INF] from='client.? 172.21.15.77:0/756362502' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:30.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:30 smithi077 bash[80681]: cluster 2024-09-13T00:47:28.610260+0000 mgr.y (mgr.25304) 348 : cluster [DBG] pgmap v644: 521 pgs: 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 211 KiB/s rd, 9.4 KiB/s wr, 258 op/s 2024-09-13T00:47:30.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:30 smithi077 bash[80681]: cluster 2024-09-13T00:47:29.056216+0000 mon.a (mon.0) 1076 : cluster [DBG] osdmap e677: 8 total, 8 up, 8 in 2024-09-13T00:47:30.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:30 smithi077 bash[80681]: audit 2024-09-13T00:47:29.075699+0000 mon.a (mon.0) 1077 : audit [INF] from='client.? 172.21.15.77:0/756362502' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:31 smithi088 bash[68960]: audit 2024-09-13T00:47:30.059893+0000 mon.a (mon.0) 1078 : audit [INF] from='client.? 172.21.15.77:0/756362502' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:47:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:31 smithi088 bash[68960]: cluster 2024-09-13T00:47:30.060204+0000 mon.a (mon.0) 1079 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2024-09-13T00:47:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:31 smithi088 bash[68960]: cluster 2024-09-13T00:47:31.007135+0000 mon.a (mon.0) 1080 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2024-09-13T00:47:31.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:31 smithi088 bash[68960]: audit 2024-09-13T00:47:31.011895+0000 mon.a (mon.0) 1081 : audit [INF] from='client.? 172.21.15.77:0/2343288226' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:31.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:31 smithi077 bash[80681]: audit 2024-09-13T00:47:30.059893+0000 mon.a (mon.0) 1078 : audit [INF] from='client.? 172.21.15.77:0/756362502' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:47:31.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:31 smithi077 bash[80681]: cluster 2024-09-13T00:47:30.060204+0000 mon.a (mon.0) 1079 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2024-09-13T00:47:31.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:31 smithi077 bash[80681]: cluster 2024-09-13T00:47:31.007135+0000 mon.a (mon.0) 1080 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2024-09-13T00:47:31.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:31 smithi077 bash[80681]: audit 2024-09-13T00:47:31.011895+0000 mon.a (mon.0) 1081 : audit [INF] from='client.? 172.21.15.77:0/2343288226' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:31.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:31 smithi077 bash[83127]: audit 2024-09-13T00:47:30.059893+0000 mon.a (mon.0) 1078 : audit [INF] from='client.? 172.21.15.77:0/756362502' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:47:31.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:31 smithi077 bash[83127]: cluster 2024-09-13T00:47:30.060204+0000 mon.a (mon.0) 1079 : cluster [DBG] osdmap e678: 8 total, 8 up, 8 in 2024-09-13T00:47:31.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:31 smithi077 bash[83127]: cluster 2024-09-13T00:47:31.007135+0000 mon.a (mon.0) 1080 : cluster [DBG] osdmap e679: 8 total, 8 up, 8 in 2024-09-13T00:47:31.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:31 smithi077 bash[83127]: audit 2024-09-13T00:47:31.011895+0000 mon.a (mon.0) 1081 : audit [INF] from='client.? 172.21.15.77:0/2343288226' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:32 smithi088 bash[68960]: cluster 2024-09-13T00:47:30.611294+0000 mgr.y (mgr.25304) 349 : cluster [DBG] pgmap v647: 553 pgs: 19 creating+peering, 13 unknown, 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 468 KiB/s rd, 12 KiB/s wr, 566 op/s 2024-09-13T00:47:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:32 smithi088 bash[68960]: audit 2024-09-13T00:47:32.012509+0000 mon.a (mon.0) 1082 : audit [INF] from='client.? 172.21.15.77:0/2343288226' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:47:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:32 smithi088 bash[68960]: cluster 2024-09-13T00:47:32.013042+0000 mon.a (mon.0) 1083 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2024-09-13T00:47:32.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:47:32 smithi088 bash[61791]: ts=2024-09-13T00:47:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:47:32.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:32 smithi077 bash[80681]: cluster 2024-09-13T00:47:30.611294+0000 mgr.y (mgr.25304) 349 : cluster [DBG] pgmap v647: 553 pgs: 19 creating+peering, 13 unknown, 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 468 KiB/s rd, 12 KiB/s wr, 566 op/s 2024-09-13T00:47:32.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:32 smithi077 bash[80681]: audit 2024-09-13T00:47:32.012509+0000 mon.a (mon.0) 1082 : audit [INF] from='client.? 172.21.15.77:0/2343288226' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:47:32.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:32 smithi077 bash[80681]: cluster 2024-09-13T00:47:32.013042+0000 mon.a (mon.0) 1083 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2024-09-13T00:47:32.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:32 smithi077 bash[83127]: cluster 2024-09-13T00:47:30.611294+0000 mgr.y (mgr.25304) 349 : cluster [DBG] pgmap v647: 553 pgs: 19 creating+peering, 13 unknown, 521 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 468 KiB/s rd, 12 KiB/s wr, 566 op/s 2024-09-13T00:47:32.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:32 smithi077 bash[83127]: audit 2024-09-13T00:47:32.012509+0000 mon.a (mon.0) 1082 : audit [INF] from='client.? 172.21.15.77:0/2343288226' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:47:32.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:32 smithi077 bash[83127]: cluster 2024-09-13T00:47:32.013042+0000 mon.a (mon.0) 1083 : cluster [DBG] osdmap e680: 8 total, 8 up, 8 in 2024-09-13T00:47:34.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:34 smithi077 bash[83127]: cluster 2024-09-13T00:47:32.612276+0000 mgr.y (mgr.25304) 350 : cluster [DBG] pgmap v650: 585 pgs: 21 creating+peering, 40 unknown, 524 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 323 KiB/s rd, 4.5 KiB/s wr, 392 op/s 2024-09-13T00:47:34.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:34 smithi077 bash[83127]: cluster 2024-09-13T00:47:33.012033+0000 mon.a (mon.0) 1084 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2024-09-13T00:47:34.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:34 smithi077 bash[83127]: audit 2024-09-13T00:47:33.015998+0000 mon.a (mon.0) 1085 : audit [INF] from='client.? 172.21.15.77:0/2111833636' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:34.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:34 smithi077 bash[83127]: audit 2024-09-13T00:47:33.217372+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:34.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:34 smithi077 bash[80681]: cluster 2024-09-13T00:47:32.612276+0000 mgr.y (mgr.25304) 350 : cluster [DBG] pgmap v650: 585 pgs: 21 creating+peering, 40 unknown, 524 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 323 KiB/s rd, 4.5 KiB/s wr, 392 op/s 2024-09-13T00:47:34.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:34 smithi077 bash[80681]: cluster 2024-09-13T00:47:33.012033+0000 mon.a (mon.0) 1084 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2024-09-13T00:47:34.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:34 smithi077 bash[80681]: audit 2024-09-13T00:47:33.015998+0000 mon.a (mon.0) 1085 : audit [INF] from='client.? 172.21.15.77:0/2111833636' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:34.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:34 smithi077 bash[80681]: audit 2024-09-13T00:47:33.217372+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:34.351 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:47:34 smithi088 bash[60519]: logger=cleanup t=2024-09-13T00:47:34.057809725Z level=info msg="Completed cleanup jobs" duration=105.001222ms 2024-09-13T00:47:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:34 smithi088 bash[68960]: cluster 2024-09-13T00:47:32.612276+0000 mgr.y (mgr.25304) 350 : cluster [DBG] pgmap v650: 585 pgs: 21 creating+peering, 40 unknown, 524 active+clean; 131 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 323 KiB/s rd, 4.5 KiB/s wr, 392 op/s 2024-09-13T00:47:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:34 smithi088 bash[68960]: cluster 2024-09-13T00:47:33.012033+0000 mon.a (mon.0) 1084 : cluster [DBG] osdmap e681: 8 total, 8 up, 8 in 2024-09-13T00:47:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:34 smithi088 bash[68960]: audit 2024-09-13T00:47:33.015998+0000 mon.a (mon.0) 1085 : audit [INF] from='client.? 172.21.15.77:0/2111833636' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:47:34.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:34 smithi088 bash[68960]: audit 2024-09-13T00:47:33.217372+0000 mon.a (mon.0) 1086 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:35.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:35 smithi077 bash[80681]: audit 2024-09-13T00:47:34.015729+0000 mon.a (mon.0) 1087 : audit [INF] from='client.? 172.21.15.77:0/2111833636' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:47:35.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:35 smithi077 bash[80681]: cluster 2024-09-13T00:47:34.016276+0000 mon.a (mon.0) 1088 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2024-09-13T00:47:35.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:35 smithi077 bash[80681]: audit 2024-09-13T00:47:34.034599+0000 mon.a (mon.0) 1089 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi077-68798-28", "tierpool":"test-librbd-smithi077-68798-30", "force_nonempty":""}]: dispatch 2024-09-13T00:47:35.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:35 smithi077 bash[83127]: audit 2024-09-13T00:47:34.015729+0000 mon.a (mon.0) 1087 : audit [INF] from='client.? 172.21.15.77:0/2111833636' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:47:35.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:35 smithi077 bash[83127]: cluster 2024-09-13T00:47:34.016276+0000 mon.a (mon.0) 1088 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2024-09-13T00:47:35.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:35 smithi077 bash[83127]: audit 2024-09-13T00:47:34.034599+0000 mon.a (mon.0) 1089 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi077-68798-28", "tierpool":"test-librbd-smithi077-68798-30", "force_nonempty":""}]: dispatch 2024-09-13T00:47:35.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:35 smithi088 bash[68960]: audit 2024-09-13T00:47:34.015729+0000 mon.a (mon.0) 1087 : audit [INF] from='client.? 172.21.15.77:0/2111833636' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:47:35.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:35 smithi088 bash[68960]: cluster 2024-09-13T00:47:34.016276+0000 mon.a (mon.0) 1088 : cluster [DBG] osdmap e682: 8 total, 8 up, 8 in 2024-09-13T00:47:35.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:35 smithi088 bash[68960]: audit 2024-09-13T00:47:34.034599+0000 mon.a (mon.0) 1089 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi077-68798-28", "tierpool":"test-librbd-smithi077-68798-30", "force_nonempty":""}]: dispatch 2024-09-13T00:47:36.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:36 smithi088 bash[68960]: cluster 2024-09-13T00:47:34.615417+0000 mgr.y (mgr.25304) 351 : cluster [DBG] pgmap v653: 617 pgs: 12 creating+peering, 23 unknown, 582 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 48 KiB/s rd, 511 B/s wr, 62 op/s 2024-09-13T00:47:36.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:36 smithi088 bash[68960]: audit 2024-09-13T00:47:35.044274+0000 mon.a (mon.0) 1090 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi077-68798-28", "tierpool":"test-librbd-smithi077-68798-30", "force_nonempty":""}]': finished 2024-09-13T00:47:36.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:36 smithi088 bash[68960]: cluster 2024-09-13T00:47:35.044439+0000 mon.a (mon.0) 1091 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2024-09-13T00:47:36.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:36 smithi088 bash[68960]: audit 2024-09-13T00:47:35.047616+0000 mon.a (mon.0) 1092 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi077-68798-30", "mode":"writeback"}]: dispatch 2024-09-13T00:47:36.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:36 smithi088 bash[68960]: cluster 2024-09-13T00:47:35.997947+0000 mon.a (mon.0) 1093 : cluster [WRN] Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-09-13T00:47:36.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:36 smithi088 bash[68960]: audit 2024-09-13T00:47:36.011266+0000 mon.a (mon.0) 1094 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi077-68798-30", "mode":"writeback"}]': finished 2024-09-13T00:47:36.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:36 smithi088 bash[68960]: cluster 2024-09-13T00:47:36.011427+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2024-09-13T00:47:36.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:36 smithi088 bash[68960]: audit 2024-09-13T00:47:36.013174+0000 mon.a (mon.0) 1096 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi077-68798-28", "overlaypool":"test-librbd-smithi077-68798-30"}]: dispatch 2024-09-13T00:47:36.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:36 smithi077 bash[80681]: cluster 2024-09-13T00:47:34.615417+0000 mgr.y (mgr.25304) 351 : cluster [DBG] pgmap v653: 617 pgs: 12 creating+peering, 23 unknown, 582 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 48 KiB/s rd, 511 B/s wr, 62 op/s 2024-09-13T00:47:36.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:36 smithi077 bash[80681]: audit 2024-09-13T00:47:35.044274+0000 mon.a (mon.0) 1090 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi077-68798-28", "tierpool":"test-librbd-smithi077-68798-30", "force_nonempty":""}]': finished 2024-09-13T00:47:36.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:36 smithi077 bash[80681]: cluster 2024-09-13T00:47:35.044439+0000 mon.a (mon.0) 1091 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2024-09-13T00:47:36.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:36 smithi077 bash[80681]: audit 2024-09-13T00:47:35.047616+0000 mon.a (mon.0) 1092 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi077-68798-30", "mode":"writeback"}]: dispatch 2024-09-13T00:47:36.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:36 smithi077 bash[80681]: cluster 2024-09-13T00:47:35.997947+0000 mon.a (mon.0) 1093 : cluster [WRN] Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-09-13T00:47:36.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:36 smithi077 bash[80681]: audit 2024-09-13T00:47:36.011266+0000 mon.a (mon.0) 1094 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi077-68798-30", "mode":"writeback"}]': finished 2024-09-13T00:47:36.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:36 smithi077 bash[80681]: cluster 2024-09-13T00:47:36.011427+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2024-09-13T00:47:36.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:36 smithi077 bash[80681]: audit 2024-09-13T00:47:36.013174+0000 mon.a (mon.0) 1096 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi077-68798-28", "overlaypool":"test-librbd-smithi077-68798-30"}]: dispatch 2024-09-13T00:47:36.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:36 smithi077 bash[83127]: cluster 2024-09-13T00:47:34.615417+0000 mgr.y (mgr.25304) 351 : cluster [DBG] pgmap v653: 617 pgs: 12 creating+peering, 23 unknown, 582 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 48 KiB/s rd, 511 B/s wr, 62 op/s 2024-09-13T00:47:36.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:36 smithi077 bash[83127]: audit 2024-09-13T00:47:35.044274+0000 mon.a (mon.0) 1090 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi077-68798-28", "tierpool":"test-librbd-smithi077-68798-30", "force_nonempty":""}]': finished 2024-09-13T00:47:36.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:36 smithi077 bash[83127]: cluster 2024-09-13T00:47:35.044439+0000 mon.a (mon.0) 1091 : cluster [DBG] osdmap e683: 8 total, 8 up, 8 in 2024-09-13T00:47:36.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:36 smithi077 bash[83127]: audit 2024-09-13T00:47:35.047616+0000 mon.a (mon.0) 1092 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi077-68798-30", "mode":"writeback"}]: dispatch 2024-09-13T00:47:36.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:36 smithi077 bash[83127]: cluster 2024-09-13T00:47:35.997947+0000 mon.a (mon.0) 1093 : cluster [WRN] Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-09-13T00:47:36.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:36 smithi077 bash[83127]: audit 2024-09-13T00:47:36.011266+0000 mon.a (mon.0) 1094 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi077-68798-30", "mode":"writeback"}]': finished 2024-09-13T00:47:36.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:36 smithi077 bash[83127]: cluster 2024-09-13T00:47:36.011427+0000 mon.a (mon.0) 1095 : cluster [DBG] osdmap e684: 8 total, 8 up, 8 in 2024-09-13T00:47:36.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:36 smithi077 bash[83127]: audit 2024-09-13T00:47:36.013174+0000 mon.a (mon.0) 1096 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi077-68798-28", "overlaypool":"test-librbd-smithi077-68798-30"}]: dispatch 2024-09-13T00:47:37.016 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:38.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:38 smithi077 bash[80681]: cluster 2024-09-13T00:47:36.617148+0000 mgr.y (mgr.25304) 352 : cluster [DBG] pgmap v656: 617 pgs: 6 creating+peering, 5 unknown, 606 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 255 B/s wr, 31 op/s 2024-09-13T00:47:38.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:38 smithi077 bash[80681]: audit 2024-09-13T00:47:37.015207+0000 mon.a (mon.0) 1097 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi077-68798-28", "overlaypool":"test-librbd-smithi077-68798-30"}]': finished 2024-09-13T00:47:38.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:38 smithi077 bash[80681]: cluster 2024-09-13T00:47:37.015342+0000 mon.a (mon.0) 1098 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2024-09-13T00:47:38.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:38 smithi077 bash[83127]: cluster 2024-09-13T00:47:36.617148+0000 mgr.y (mgr.25304) 352 : cluster [DBG] pgmap v656: 617 pgs: 6 creating+peering, 5 unknown, 606 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 255 B/s wr, 31 op/s 2024-09-13T00:47:38.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:38 smithi077 bash[83127]: audit 2024-09-13T00:47:37.015207+0000 mon.a (mon.0) 1097 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi077-68798-28", "overlaypool":"test-librbd-smithi077-68798-30"}]': finished 2024-09-13T00:47:38.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:38 smithi077 bash[83127]: cluster 2024-09-13T00:47:37.015342+0000 mon.a (mon.0) 1098 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2024-09-13T00:47:38.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:38 smithi088 bash[68960]: cluster 2024-09-13T00:47:36.617148+0000 mgr.y (mgr.25304) 352 : cluster [DBG] pgmap v656: 617 pgs: 6 creating+peering, 5 unknown, 606 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 255 B/s wr, 31 op/s 2024-09-13T00:47:38.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:38 smithi088 bash[68960]: audit 2024-09-13T00:47:37.015207+0000 mon.a (mon.0) 1097 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi077-68798-28", "overlaypool":"test-librbd-smithi077-68798-30"}]': finished 2024-09-13T00:47:38.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:38 smithi088 bash[68960]: cluster 2024-09-13T00:47:37.015342+0000 mon.a (mon.0) 1098 : cluster [DBG] osdmap e685: 8 total, 8 up, 8 in 2024-09-13T00:47:38.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:47:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:47:38] "GET /metrics HTTP/1.1" 200 67895 "" "Prometheus/2.43.0" 2024-09-13T00:47:39.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:39 smithi088 bash[68960]: cluster 2024-09-13T00:47:38.052741+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2024-09-13T00:47:39.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:39 smithi077 bash[80681]: cluster 2024-09-13T00:47:38.052741+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2024-09-13T00:47:39.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:39 smithi077 bash[83127]: cluster 2024-09-13T00:47:38.052741+0000 mon.a (mon.0) 1099 : cluster [DBG] osdmap e686: 8 total, 8 up, 8 in 2024-09-13T00:47:40.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:40 smithi088 bash[68960]: cluster 2024-09-13T00:47:38.618924+0000 mgr.y (mgr.25304) 353 : cluster [DBG] pgmap v659: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:47:40.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:40 smithi088 bash[68960]: cluster 2024-09-13T00:47:39.062895+0000 mon.a (mon.0) 1100 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2024-09-13T00:47:40.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:40 smithi077 bash[80681]: cluster 2024-09-13T00:47:38.618924+0000 mgr.y (mgr.25304) 353 : cluster [DBG] pgmap v659: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:47:40.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:40 smithi077 bash[80681]: cluster 2024-09-13T00:47:39.062895+0000 mon.a (mon.0) 1100 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2024-09-13T00:47:40.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:40 smithi077 bash[83127]: cluster 2024-09-13T00:47:38.618924+0000 mgr.y (mgr.25304) 353 : cluster [DBG] pgmap v659: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:47:40.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:40 smithi077 bash[83127]: cluster 2024-09-13T00:47:39.062895+0000 mon.a (mon.0) 1100 : cluster [DBG] osdmap e687: 8 total, 8 up, 8 in 2024-09-13T00:47:41.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:41 smithi088 bash[68960]: cluster 2024-09-13T00:47:40.068831+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2024-09-13T00:47:41.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:41 smithi077 bash[83127]: cluster 2024-09-13T00:47:40.068831+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2024-09-13T00:47:41.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:41 smithi077 bash[80681]: cluster 2024-09-13T00:47:40.068831+0000 mon.a (mon.0) 1101 : cluster [DBG] osdmap e688: 8 total, 8 up, 8 in 2024-09-13T00:47:42.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:42 smithi088 bash[68960]: cluster 2024-09-13T00:47:40.620058+0000 mgr.y (mgr.25304) 354 : cluster [DBG] pgmap v662: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-13T00:47:42.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:42 smithi088 bash[68960]: cluster 2024-09-13T00:47:41.092729+0000 mon.a (mon.0) 1102 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2024-09-13T00:47:42.352 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:47:42 smithi088 bash[61791]: ts=2024-09-13T00:47:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:47:42.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:42 smithi077 bash[83127]: cluster 2024-09-13T00:47:40.620058+0000 mgr.y (mgr.25304) 354 : cluster [DBG] pgmap v662: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-13T00:47:42.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:42 smithi077 bash[83127]: cluster 2024-09-13T00:47:41.092729+0000 mon.a (mon.0) 1102 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2024-09-13T00:47:42.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:42 smithi077 bash[80681]: cluster 2024-09-13T00:47:40.620058+0000 mgr.y (mgr.25304) 354 : cluster [DBG] pgmap v662: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 KiB/s wr, 4 op/s 2024-09-13T00:47:42.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:42 smithi077 bash[80681]: cluster 2024-09-13T00:47:41.092729+0000 mon.a (mon.0) 1102 : cluster [DBG] osdmap e689: 8 total, 8 up, 8 in 2024-09-13T00:47:43.460 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.460 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.461 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.461 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.461 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.461 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 1 2024-09-13T00:47:43.461 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 1 2024-09-13T00:47:43.461 INFO:tasks.workunit.client.0.smithi077.stdout:child: image101 2024-09-13T00:47:43.462 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image101/85feb5fb321f 2024-09-13T00:47:43.462 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.462 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.462 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.462 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.462 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.462 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.462 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.462 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.463 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.463 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.463 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.463 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.463 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 2 2024-09-13T00:47:43.463 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 2 2024-09-13T00:47:43.463 INFO:tasks.workunit.client.0.smithi077.stdout:child: image102 2024-09-13T00:47:43.463 INFO:tasks.workunit.client.0.smithi077.stdout:child: image101 2024-09-13T00:47:43.463 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image101/85feb5fb321f 2024-09-13T00:47:43.463 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.463 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-28/image102/85fe2b998495 2024-09-13T00:47:43.463 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.463 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.464 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.464 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.464 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.464 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.464 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.464 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.464 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.464 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.464 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.464 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.464 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.464 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-29/image103 2024-09-13T00:47:43.464 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.465 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.465 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image103 2024-09-13T00:47:43.465 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image103 2024-09-13T00:47:43.465 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.465 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 3 2024-09-13T00:47:43.465 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 3 2024-09-13T00:47:43.465 INFO:tasks.workunit.client.0.smithi077.stdout:child: image102 2024-09-13T00:47:43.466 INFO:tasks.workunit.client.0.smithi077.stdout:child: image101 2024-09-13T00:47:43.466 INFO:tasks.workunit.client.0.smithi077.stdout:child: image103 2024-09-13T00:47:43.466 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image101/85feb5fb321f 2024-09-13T00:47:43.466 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.466 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-28/image102/85fe2b998495 2024-09-13T00:47:43.467 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.467 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image103/85fecdc1f55b 2024-09-13T00:47:43.467 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.467 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.467 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.467 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.468 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.468 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.468 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.468 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.468 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.469 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.469 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.469 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.469 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 3 2024-09-13T00:47:43.469 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 3 2024-09-13T00:47:43.469 INFO:tasks.workunit.client.0.smithi077.stdout:child: image102 2024-09-13T00:47:43.470 INFO:tasks.workunit.client.0.smithi077.stdout:child: image101 2024-09-13T00:47:43.470 INFO:tasks.workunit.client.0.smithi077.stdout:child: image103 2024-09-13T00:47:43.470 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image101/85feb5fb321f 2024-09-13T00:47:43.471 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.471 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-28/image102/85fe2b998495 2024-09-13T00:47:43.471 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.471 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image103/85fecdc1f55b 2024-09-13T00:47:43.471 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.471 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.471 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.472 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.472 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.472 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.472 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.472 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.472 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.472 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.472 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.472 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.472 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.472 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-29/image104 2024-09-13T00:47:43.472 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.473 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.473 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image104 2024-09-13T00:47:43.473 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image104 2024-09-13T00:47:43.473 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.473 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 4 2024-09-13T00:47:43.473 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 4 2024-09-13T00:47:43.473 INFO:tasks.workunit.client.0.smithi077.stdout:child: image102 2024-09-13T00:47:43.473 INFO:tasks.workunit.client.0.smithi077.stdout:child: image101 2024-09-13T00:47:43.473 INFO:tasks.workunit.client.0.smithi077.stdout:child: image103 2024-09-13T00:47:43.474 INFO:tasks.workunit.client.0.smithi077.stdout:child: image104 2024-09-13T00:47:43.474 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image101/85feb5fb321f 2024-09-13T00:47:43.474 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.474 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-28/image102/85fe2b998495 2024-09-13T00:47:43.474 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.474 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image103/85fecdc1f55b 2024-09-13T00:47:43.474 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.474 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image104/85feb184f185 2024-09-13T00:47:43.474 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.474 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.474 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.474 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.474 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.474 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.475 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.475 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.475 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.475 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.475 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.475 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.475 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.475 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-29/image103 2024-09-13T00:47:43.475 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.475 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:43.476 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image103 2024-09-13T00:47:43.476 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image103 2024-09-13T00:47:43.476 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.476 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:43.476 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-29/image104 2024-09-13T00:47:43.476 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:43.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:43 smithi077 bash[80681]: cluster 2024-09-13T00:47:42.104113+0000 mon.a (mon.0) 1103 : cluster [DBG] osdmap e690: 8 total, 8 up, 8 in 2024-09-13T00:47:43.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:43 smithi077 bash[83127]: cluster 2024-09-13T00:47:42.104113+0000 mon.a (mon.0) 1103 : cluster [DBG] osdmap e690: 8 total, 8 up, 8 in 2024-09-13T00:47:43.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:43 smithi088 bash[68960]: cluster 2024-09-13T00:47:42.104113+0000 mon.a (mon.0) 1103 : cluster [DBG] osdmap e690: 8 total, 8 up, 8 in 2024-09-13T00:47:44.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:44 smithi088 bash[68960]: cluster 2024-09-13T00:47:42.621465+0000 mgr.y (mgr.25304) 355 : cluster [DBG] pgmap v665: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 KiB/s wr, 39 op/s 2024-09-13T00:47:44.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:44 smithi088 bash[68960]: audit 2024-09-13T00:47:43.286478+0000 mon.a (mon.0) 1104 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:44.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:44 smithi088 bash[68960]: audit 2024-09-13T00:47:43.430339+0000 mon.a (mon.0) 1105 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:44.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:44 smithi088 bash[68960]: audit 2024-09-13T00:47:43.433225+0000 mon.a (mon.0) 1106 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:44.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:44 smithi077 bash[83127]: cluster 2024-09-13T00:47:42.621465+0000 mgr.y (mgr.25304) 355 : cluster [DBG] pgmap v665: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 KiB/s wr, 39 op/s 2024-09-13T00:47:44.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:44 smithi077 bash[83127]: audit 2024-09-13T00:47:43.286478+0000 mon.a (mon.0) 1104 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:44.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:44 smithi077 bash[83127]: audit 2024-09-13T00:47:43.430339+0000 mon.a (mon.0) 1105 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:44.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:44 smithi077 bash[83127]: audit 2024-09-13T00:47:43.433225+0000 mon.a (mon.0) 1106 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:44.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:44 smithi077 bash[80681]: cluster 2024-09-13T00:47:42.621465+0000 mgr.y (mgr.25304) 355 : cluster [DBG] pgmap v665: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 KiB/s wr, 39 op/s 2024-09-13T00:47:44.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:44 smithi077 bash[80681]: audit 2024-09-13T00:47:43.286478+0000 mon.a (mon.0) 1104 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:44.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:44 smithi077 bash[80681]: audit 2024-09-13T00:47:43.430339+0000 mon.a (mon.0) 1105 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:44.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:44 smithi077 bash[80681]: audit 2024-09-13T00:47:43.433225+0000 mon.a (mon.0) 1106 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:46.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:46 smithi088 bash[68960]: cluster 2024-09-13T00:47:44.625951+0000 mgr.y (mgr.25304) 356 : cluster [DBG] pgmap v666: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 244 KiB/s rd, 6.1 KiB/s wr, 291 op/s 2024-09-13T00:47:46.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:46 smithi088 bash[68960]: cluster 2024-09-13T00:47:45.298955+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e691: 8 total, 8 up, 8 in 2024-09-13T00:47:46.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:46 smithi088 bash[68960]: audit 2024-09-13T00:47:45.492045+0000 mon.a (mon.0) 1108 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi077-68798-28"}]: dispatch 2024-09-13T00:47:46.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:46 smithi088 bash[68960]: audit 2024-09-13T00:47:46.015739+0000 mon.a (mon.0) 1109 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi077-68798-28"}]': finished 2024-09-13T00:47:46.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:46 smithi088 bash[68960]: cluster 2024-09-13T00:47:46.015888+0000 mon.a (mon.0) 1110 : cluster [DBG] osdmap e692: 8 total, 8 up, 8 in 2024-09-13T00:47:46.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:46 smithi088 bash[68960]: audit 2024-09-13T00:47:46.019676+0000 mon.a (mon.0) 1111 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi077-68798-28", "tierpool":"test-librbd-smithi077-68798-30"}]: dispatch 2024-09-13T00:47:46.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:46 smithi077 bash[80681]: cluster 2024-09-13T00:47:44.625951+0000 mgr.y (mgr.25304) 356 : cluster [DBG] pgmap v666: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 244 KiB/s rd, 6.1 KiB/s wr, 291 op/s 2024-09-13T00:47:46.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:46 smithi077 bash[80681]: cluster 2024-09-13T00:47:45.298955+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e691: 8 total, 8 up, 8 in 2024-09-13T00:47:46.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:46 smithi077 bash[80681]: audit 2024-09-13T00:47:45.492045+0000 mon.a (mon.0) 1108 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi077-68798-28"}]: dispatch 2024-09-13T00:47:46.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:46 smithi077 bash[80681]: audit 2024-09-13T00:47:46.015739+0000 mon.a (mon.0) 1109 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi077-68798-28"}]': finished 2024-09-13T00:47:46.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:46 smithi077 bash[80681]: cluster 2024-09-13T00:47:46.015888+0000 mon.a (mon.0) 1110 : cluster [DBG] osdmap e692: 8 total, 8 up, 8 in 2024-09-13T00:47:46.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:46 smithi077 bash[80681]: audit 2024-09-13T00:47:46.019676+0000 mon.a (mon.0) 1111 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi077-68798-28", "tierpool":"test-librbd-smithi077-68798-30"}]: dispatch 2024-09-13T00:47:46.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:46 smithi077 bash[83127]: cluster 2024-09-13T00:47:44.625951+0000 mgr.y (mgr.25304) 356 : cluster [DBG] pgmap v666: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 244 KiB/s rd, 6.1 KiB/s wr, 291 op/s 2024-09-13T00:47:46.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:46 smithi077 bash[83127]: cluster 2024-09-13T00:47:45.298955+0000 mon.a (mon.0) 1107 : cluster [DBG] osdmap e691: 8 total, 8 up, 8 in 2024-09-13T00:47:46.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:46 smithi077 bash[83127]: audit 2024-09-13T00:47:45.492045+0000 mon.a (mon.0) 1108 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi077-68798-28"}]: dispatch 2024-09-13T00:47:46.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:46 smithi077 bash[83127]: audit 2024-09-13T00:47:46.015739+0000 mon.a (mon.0) 1109 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi077-68798-28"}]': finished 2024-09-13T00:47:46.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:46 smithi077 bash[83127]: cluster 2024-09-13T00:47:46.015888+0000 mon.a (mon.0) 1110 : cluster [DBG] osdmap e692: 8 total, 8 up, 8 in 2024-09-13T00:47:46.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:46 smithi077 bash[83127]: audit 2024-09-13T00:47:46.019676+0000 mon.a (mon.0) 1111 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi077-68798-28", "tierpool":"test-librbd-smithi077-68798-30"}]: dispatch 2024-09-13T00:47:47.016 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image101 2024-09-13T00:47:47.016 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image103 2024-09-13T00:47:47.016 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image104 2024-09-13T00:47:47.016 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image104 2024-09-13T00:47:47.017 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.017 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 4 2024-09-13T00:47:47.017 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 4 2024-09-13T00:47:47.017 INFO:tasks.workunit.client.0.smithi077.stdout:child: image102 2024-09-13T00:47:47.017 INFO:tasks.workunit.client.0.smithi077.stdout:child: image101 2024-09-13T00:47:47.017 INFO:tasks.workunit.client.0.smithi077.stdout:child: image103 2024-09-13T00:47:47.017 INFO:tasks.workunit.client.0.smithi077.stdout:child: image104 2024-09-13T00:47:47.017 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image101/85feb5fb321f 2024-09-13T00:47:47.017 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.017 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-28/image102/85fe2b998495 2024-09-13T00:47:47.017 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.017 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image103/85fecdc1f55b 2024-09-13T00:47:47.017 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.018 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image104/85feb184f185 2024-09-13T00:47:47.018 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.018 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.018 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:47.018 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:47.018 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:47.018 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.018 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.018 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-29/image103 2024-09-13T00:47:47.019 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:47.019 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image103 2024-09-13T00:47:47.019 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image103 2024-09-13T00:47:47.019 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.019 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.019 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-29/image104 2024-09-13T00:47:47.019 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:47.019 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image103 2024-09-13T00:47:47.019 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image104 2024-09-13T00:47:47.020 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image104 2024-09-13T00:47:47.020 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.020 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 3 2024-09-13T00:47:47.020 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 3 2024-09-13T00:47:47.020 INFO:tasks.workunit.client.0.smithi077.stdout:child: image102 2024-09-13T00:47:47.020 INFO:tasks.workunit.client.0.smithi077.stdout:child: image103 2024-09-13T00:47:47.020 INFO:tasks.workunit.client.0.smithi077.stdout:child: image104 2024-09-13T00:47:47.020 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-28/image102/85fe2b998495 2024-09-13T00:47:47.020 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.020 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image103/85fecdc1f55b 2024-09-13T00:47:47.020 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.020 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image104/85feb184f185 2024-09-13T00:47:47.020 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.021 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.021 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:47.021 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:47.021 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:47.021 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.021 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.021 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-29/image104 2024-09-13T00:47:47.021 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:47.021 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-29/image104 2024-09-13T00:47:47.021 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image104 2024-09-13T00:47:47.021 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.021 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 2 2024-09-13T00:47:47.021 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 2 2024-09-13T00:47:47.022 INFO:tasks.workunit.client.0.smithi077.stdout:child: image102 2024-09-13T00:47:47.022 INFO:tasks.workunit.client.0.smithi077.stdout:child: image104 2024-09-13T00:47:47.022 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-28/image102/85fe2b998495 2024-09-13T00:47:47.022 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.022 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-29/image104/85feb184f185 2024-09-13T00:47:47.022 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.022 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.022 INFO:tasks.workunit.client.0.smithi077.stdout:trying to find test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:47.022 INFO:tasks.workunit.client.0.smithi077.stdout:checking test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:47.022 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-28/image102 2024-09-13T00:47:47.022 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.022 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 1 2024-09-13T00:47:47.022 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 1 2024-09-13T00:47:47.022 INFO:tasks.workunit.client.0.smithi077.stdout:child: image102 2024-09-13T00:47:47.023 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-28/image102/85fe2b998495 2024-09-13T00:47:47.023 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:47:47.023 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 0 2024-09-13T00:47:47.023 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 0 2024-09-13T00:47:47.023 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ListChildrenTiered (18771 ms) 2024-09-13T00:47:47.023 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.LockingPP 2024-09-13T00:47:47.023 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:47.193 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.LockingPP (177 ms) 2024-09-13T00:47:47.193 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.FlushAio 2024-09-13T00:47:47.194 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:47.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:47 smithi088 bash[68960]: cluster 2024-09-13T00:47:47.010215+0000 mon.a (mon.0) 1112 : cluster [INF] Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-09-13T00:47:47.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:47 smithi088 bash[68960]: audit 2024-09-13T00:47:47.019284+0000 mon.a (mon.0) 1113 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi077-68798-28", "tierpool":"test-librbd-smithi077-68798-30"}]': finished 2024-09-13T00:47:47.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:47 smithi088 bash[68960]: cluster 2024-09-13T00:47:47.019442+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e693: 8 total, 8 up, 8 in 2024-09-13T00:47:47.653 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.FlushAio (460 ms) 2024-09-13T00:47:47.653 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.FlushAioPP 2024-09-13T00:47:47.653 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:47.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:47 smithi077 bash[80681]: cluster 2024-09-13T00:47:47.010215+0000 mon.a (mon.0) 1112 : cluster [INF] Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-09-13T00:47:47.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:47 smithi077 bash[80681]: audit 2024-09-13T00:47:47.019284+0000 mon.a (mon.0) 1113 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi077-68798-28", "tierpool":"test-librbd-smithi077-68798-30"}]': finished 2024-09-13T00:47:47.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:47 smithi077 bash[80681]: cluster 2024-09-13T00:47:47.019442+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e693: 8 total, 8 up, 8 in 2024-09-13T00:47:47.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:47 smithi077 bash[83127]: cluster 2024-09-13T00:47:47.010215+0000 mon.a (mon.0) 1112 : cluster [INF] Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-09-13T00:47:47.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:47 smithi077 bash[83127]: audit 2024-09-13T00:47:47.019284+0000 mon.a (mon.0) 1113 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi077-68798-28", "tierpool":"test-librbd-smithi077-68798-30"}]': finished 2024-09-13T00:47:47.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:47 smithi077 bash[83127]: cluster 2024-09-13T00:47:47.019442+0000 mon.a (mon.0) 1114 : cluster [DBG] osdmap e693: 8 total, 8 up, 8 in 2024-09-13T00:47:47.950 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.FlushAioPP (297 ms) 2024-09-13T00:47:47.951 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.SnapDiff 2024-09-13T00:47:47.951 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:48.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:48 smithi088 bash[68960]: cluster 2024-09-13T00:47:46.627720+0000 mgr.y (mgr.25304) 357 : cluster [DBG] pgmap v669: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 342 KiB/s rd, 9.4 KiB/s wr, 415 op/s 2024-09-13T00:47:48.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:48 smithi077 bash[80681]: cluster 2024-09-13T00:47:46.627720+0000 mgr.y (mgr.25304) 357 : cluster [DBG] pgmap v669: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 342 KiB/s rd, 9.4 KiB/s wr, 415 op/s 2024-09-13T00:47:48.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:48 smithi077 bash[83127]: cluster 2024-09-13T00:47:46.627720+0000 mgr.y (mgr.25304) 357 : cluster [DBG] pgmap v669: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 342 KiB/s rd, 9.4 KiB/s wr, 415 op/s 2024-09-13T00:47:48.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:47:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:47:48] "GET /metrics HTTP/1.1" 200 75140 "" "Prometheus/2.43.0" 2024-09-13T00:47:49.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:49 smithi088 bash[68960]: cluster 2024-09-13T00:47:48.320416+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e694: 8 total, 8 up, 8 in 2024-09-13T00:47:49.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:49 smithi077 bash[80681]: cluster 2024-09-13T00:47:48.320416+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e694: 8 total, 8 up, 8 in 2024-09-13T00:47:49.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:49 smithi077 bash[83127]: cluster 2024-09-13T00:47:48.320416+0000 mon.a (mon.0) 1115 : cluster [DBG] osdmap e694: 8 total, 8 up, 8 in 2024-09-13T00:47:50.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:50 smithi088 bash[68960]: cluster 2024-09-13T00:47:48.629380+0000 mgr.y (mgr.25304) 358 : cluster [DBG] pgmap v672: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 266 KiB/s rd, 7.2 KiB/s wr, 332 op/s 2024-09-13T00:47:50.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:50 smithi088 bash[68960]: cluster 2024-09-13T00:47:49.330028+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e695: 8 total, 8 up, 8 in 2024-09-13T00:47:50.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:50 smithi077 bash[80681]: cluster 2024-09-13T00:47:48.629380+0000 mgr.y (mgr.25304) 358 : cluster [DBG] pgmap v672: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 266 KiB/s rd, 7.2 KiB/s wr, 332 op/s 2024-09-13T00:47:50.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:50 smithi077 bash[80681]: cluster 2024-09-13T00:47:49.330028+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e695: 8 total, 8 up, 8 in 2024-09-13T00:47:50.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:50 smithi077 bash[83127]: cluster 2024-09-13T00:47:48.629380+0000 mgr.y (mgr.25304) 358 : cluster [DBG] pgmap v672: 617 pgs: 617 active+clean; 131 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 266 KiB/s rd, 7.2 KiB/s wr, 332 op/s 2024-09-13T00:47:50.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:50 smithi077 bash[83127]: cluster 2024-09-13T00:47:49.330028+0000 mon.a (mon.0) 1116 : cluster [DBG] osdmap e695: 8 total, 8 up, 8 in 2024-09-13T00:47:51.271 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:47:51.271 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.SnapDiff (3320 ms) 2024-09-13T00:47:51.271 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ZeroLengthWrite 2024-09-13T00:47:51.271 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:51.329 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ZeroLengthWrite (58 ms) 2024-09-13T00:47:51.329 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ZeroLengthDiscard 2024-09-13T00:47:51.329 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:51.403 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ZeroLengthDiscard (74 ms) 2024-09-13T00:47:51.403 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ZeroLengthRead 2024-09-13T00:47:51.403 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:51.446 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ZeroLengthRead (43 ms) 2024-09-13T00:47:51.446 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.LargeCacheRead 2024-09-13T00:47:51.446 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:51.529 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.LargeCacheRead (84 ms) 2024-09-13T00:47:51.530 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestPendingAio 2024-09-13T00:47:51.530 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:51 smithi088 bash[68960]: cluster 2024-09-13T00:47:50.343570+0000 mon.a (mon.0) 1117 : cluster [DBG] osdmap e696: 8 total, 8 up, 8 in 2024-09-13T00:47:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:51 smithi088 bash[68960]: cluster 2024-09-13T00:47:51.020445+0000 mon.a (mon.0) 1118 : cluster [DBG] osdmap e697: 8 total, 8 up, 8 in 2024-09-13T00:47:51.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:51 smithi077 bash[80681]: cluster 2024-09-13T00:47:50.343570+0000 mon.a (mon.0) 1117 : cluster [DBG] osdmap e696: 8 total, 8 up, 8 in 2024-09-13T00:47:51.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:51 smithi077 bash[80681]: cluster 2024-09-13T00:47:51.020445+0000 mon.a (mon.0) 1118 : cluster [DBG] osdmap e697: 8 total, 8 up, 8 in 2024-09-13T00:47:51.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:51 smithi077 bash[83127]: cluster 2024-09-13T00:47:50.343570+0000 mon.a (mon.0) 1117 : cluster [DBG] osdmap e696: 8 total, 8 up, 8 in 2024-09-13T00:47:51.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:51 smithi077 bash[83127]: cluster 2024-09-13T00:47:51.020445+0000 mon.a (mon.0) 1118 : cluster [DBG] osdmap e697: 8 total, 8 up, 8 in 2024-09-13T00:47:52.092 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestPendingAio (562 ms) 2024-09-13T00:47:52.092 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyup 2024-09-13T00:47:52.092 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:52.598 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:52.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:52 smithi088 bash[68960]: cluster 2024-09-13T00:47:50.630461+0000 mgr.y (mgr.25304) 359 : cluster [DBG] pgmap v675: 617 pgs: 617 active+clean; 133 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 216 KiB/s rd, 138 KiB/s wr, 409 op/s 2024-09-13T00:47:52.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:47:52 smithi088 bash[61791]: ts=2024-09-13T00:47:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:47:52.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:52 smithi077 bash[80681]: cluster 2024-09-13T00:47:50.630461+0000 mgr.y (mgr.25304) 359 : cluster [DBG] pgmap v675: 617 pgs: 617 active+clean; 133 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 216 KiB/s rd, 138 KiB/s wr, 409 op/s 2024-09-13T00:47:52.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:52 smithi077 bash[83127]: cluster 2024-09-13T00:47:50.630461+0000 mgr.y (mgr.25304) 359 : cluster [DBG] pgmap v675: 617 pgs: 617 active+clean; 133 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 216 KiB/s rd, 138 KiB/s wr, 409 op/s 2024-09-13T00:47:53.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:53 smithi077 bash[80681]: cluster 2024-09-13T00:47:52.369599+0000 mon.a (mon.0) 1119 : cluster [DBG] osdmap e698: 8 total, 8 up, 8 in 2024-09-13T00:47:53.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:53 smithi077 bash[83127]: cluster 2024-09-13T00:47:52.369599+0000 mon.a (mon.0) 1119 : cluster [DBG] osdmap e698: 8 total, 8 up, 8 in 2024-09-13T00:47:53.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:53 smithi088 bash[68960]: cluster 2024-09-13T00:47:52.369599+0000 mon.a (mon.0) 1119 : cluster [DBG] osdmap e698: 8 total, 8 up, 8 in 2024-09-13T00:47:54.494 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyup (2402 ms) 2024-09-13T00:47:54.494 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyupMismatch 2024-09-13T00:47:54.494 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:54.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:54 smithi077 bash[80681]: cluster 2024-09-13T00:47:52.631819+0000 mgr.y (mgr.25304) 360 : cluster [DBG] pgmap v678: 617 pgs: 617 active+clean; 133 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 136 KiB/s wr, 252 op/s 2024-09-13T00:47:54.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:54 smithi077 bash[80681]: cluster 2024-09-13T00:47:53.380355+0000 mon.a (mon.0) 1120 : cluster [DBG] osdmap e699: 8 total, 8 up, 8 in 2024-09-13T00:47:54.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:54 smithi077 bash[80681]: audit 2024-09-13T00:47:53.440242+0000 mon.a (mon.0) 1121 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:47:54.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:54 smithi077 bash[83127]: cluster 2024-09-13T00:47:52.631819+0000 mgr.y (mgr.25304) 360 : cluster [DBG] pgmap v678: 617 pgs: 617 active+clean; 133 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 136 KiB/s wr, 252 op/s 2024-09-13T00:47:54.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:54 smithi077 bash[83127]: cluster 2024-09-13T00:47:53.380355+0000 mon.a (mon.0) 1120 : cluster [DBG] osdmap e699: 8 total, 8 up, 8 in 2024-09-13T00:47:54.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:54 smithi077 bash[83127]: audit 2024-09-13T00:47:53.440242+0000 mon.a (mon.0) 1121 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:47:54.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:54 smithi088 bash[68960]: cluster 2024-09-13T00:47:52.631819+0000 mgr.y (mgr.25304) 360 : cluster [DBG] pgmap v678: 617 pgs: 617 active+clean; 133 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 136 KiB/s wr, 252 op/s 2024-09-13T00:47:54.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:54 smithi088 bash[68960]: cluster 2024-09-13T00:47:53.380355+0000 mon.a (mon.0) 1120 : cluster [DBG] osdmap e699: 8 total, 8 up, 8 in 2024-09-13T00:47:54.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:54 smithi088 bash[68960]: audit 2024-09-13T00:47:53.440242+0000 mon.a (mon.0) 1121 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:47:55.620 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:55.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:55 smithi077 bash[80681]: cluster 2024-09-13T00:47:54.389949+0000 mon.a (mon.0) 1122 : cluster [DBG] osdmap e700: 8 total, 8 up, 8 in 2024-09-13T00:47:55.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:55 smithi077 bash[83127]: cluster 2024-09-13T00:47:54.389949+0000 mon.a (mon.0) 1122 : cluster [DBG] osdmap e700: 8 total, 8 up, 8 in 2024-09-13T00:47:55.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:55 smithi088 bash[68960]: cluster 2024-09-13T00:47:54.389949+0000 mon.a (mon.0) 1122 : cluster [DBG] osdmap e700: 8 total, 8 up, 8 in 2024-09-13T00:47:56.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:56 smithi077 bash[80681]: cluster 2024-09-13T00:47:54.634570+0000 mgr.y (mgr.25304) 361 : cluster [DBG] pgmap v681: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 740 KiB/s rd, 595 KiB/s wr, 361 op/s 2024-09-13T00:47:56.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:56 smithi077 bash[80681]: cluster 2024-09-13T00:47:55.403476+0000 mon.a (mon.0) 1123 : cluster [DBG] osdmap e701: 8 total, 8 up, 8 in 2024-09-13T00:47:56.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:56 smithi077 bash[80681]: cluster 2024-09-13T00:47:56.022214+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e702: 8 total, 8 up, 8 in 2024-09-13T00:47:56.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:56 smithi077 bash[83127]: cluster 2024-09-13T00:47:54.634570+0000 mgr.y (mgr.25304) 361 : cluster [DBG] pgmap v681: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 740 KiB/s rd, 595 KiB/s wr, 361 op/s 2024-09-13T00:47:56.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:56 smithi077 bash[83127]: cluster 2024-09-13T00:47:55.403476+0000 mon.a (mon.0) 1123 : cluster [DBG] osdmap e701: 8 total, 8 up, 8 in 2024-09-13T00:47:56.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:56 smithi077 bash[83127]: cluster 2024-09-13T00:47:56.022214+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e702: 8 total, 8 up, 8 in 2024-09-13T00:47:56.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:56 smithi088 bash[68960]: cluster 2024-09-13T00:47:54.634570+0000 mgr.y (mgr.25304) 361 : cluster [DBG] pgmap v681: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 740 KiB/s rd, 595 KiB/s wr, 361 op/s 2024-09-13T00:47:56.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:56 smithi088 bash[68960]: cluster 2024-09-13T00:47:55.403476+0000 mon.a (mon.0) 1123 : cluster [DBG] osdmap e701: 8 total, 8 up, 8 in 2024-09-13T00:47:56.851 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:56 smithi088 bash[68960]: cluster 2024-09-13T00:47:56.022214+0000 mon.a (mon.0) 1124 : cluster [DBG] osdmap e702: 8 total, 8 up, 8 in 2024-09-13T00:47:57.114 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyupMismatch (2620 ms) 2024-09-13T00:47:57.114 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.Flatten 2024-09-13T00:47:57.114 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:58.236 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.Flatten (1122 ms) 2024-09-13T00:47:58.236 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.Sparsify 2024-09-13T00:47:58.236 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:58.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:58 smithi077 bash[80681]: cluster 2024-09-13T00:47:56.636329+0000 mgr.y (mgr.25304) 362 : cluster [DBG] pgmap v684: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 785 KiB/s rd, 601 KiB/s wr, 429 op/s 2024-09-13T00:47:58.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:58 smithi077 bash[80681]: cluster 2024-09-13T00:47:57.025535+0000 mon.a (mon.0) 1125 : cluster [DBG] osdmap e703: 8 total, 8 up, 8 in 2024-09-13T00:47:58.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:58 smithi077 bash[83127]: cluster 2024-09-13T00:47:56.636329+0000 mgr.y (mgr.25304) 362 : cluster [DBG] pgmap v684: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 785 KiB/s rd, 601 KiB/s wr, 429 op/s 2024-09-13T00:47:58.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:58 smithi077 bash[83127]: cluster 2024-09-13T00:47:57.025535+0000 mon.a (mon.0) 1125 : cluster [DBG] osdmap e703: 8 total, 8 up, 8 in 2024-09-13T00:47:58.325 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 32768 2024-09-13T00:47:58.325 INFO:tasks.workunit.client.0.smithi077.stdout:read: 32768 2024-09-13T00:47:58.325 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.Sparsify (90 ms) 2024-09-13T00:47:58.326 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.SparsifyPP 2024-09-13T00:47:58.326 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:58.350 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:58 smithi088 bash[68960]: cluster 2024-09-13T00:47:56.636329+0000 mgr.y (mgr.25304) 362 : cluster [DBG] pgmap v684: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 785 KiB/s rd, 601 KiB/s wr, 429 op/s 2024-09-13T00:47:58.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:58 smithi088 bash[68960]: cluster 2024-09-13T00:47:57.025535+0000 mon.a (mon.0) 1125 : cluster [DBG] osdmap e703: 8 total, 8 up, 8 in 2024-09-13T00:47:58.386 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.SparsifyPP (60 ms) 2024-09-13T00:47:58.386 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.SnapshotLimit 2024-09-13T00:47:58.386 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:47:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:47:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:47:58] "GET /metrics HTTP/1.1" 200 75158 "" "Prometheus/2.43.0" 2024-09-13T00:47:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:59 smithi088 bash[68960]: cluster 2024-09-13T00:47:58.038449+0000 mon.a (mon.0) 1126 : cluster [DBG] osdmap e704: 8 total, 8 up, 8 in 2024-09-13T00:47:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:59 smithi088 bash[68960]: audit 2024-09-13T00:47:58.423775+0000 mon.a (mon.0) 1127 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:59 smithi088 bash[68960]: audit 2024-09-13T00:47:58.425090+0000 mon.a (mon.0) 1128 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:59 smithi088 bash[68960]: audit 2024-09-13T00:47:59.012010+0000 mon.a (mon.0) 1129 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:59.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:47:59 smithi088 bash[68960]: audit 2024-09-13T00:47:59.019813+0000 mon.a (mon.0) 1130 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:59 smithi077 bash[80681]: cluster 2024-09-13T00:47:58.038449+0000 mon.a (mon.0) 1126 : cluster [DBG] osdmap e704: 8 total, 8 up, 8 in 2024-09-13T00:47:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:59 smithi077 bash[80681]: audit 2024-09-13T00:47:58.423775+0000 mon.a (mon.0) 1127 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:59 smithi077 bash[80681]: audit 2024-09-13T00:47:58.425090+0000 mon.a (mon.0) 1128 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:59 smithi077 bash[80681]: audit 2024-09-13T00:47:59.012010+0000 mon.a (mon.0) 1129 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:47:59 smithi077 bash[80681]: audit 2024-09-13T00:47:59.019813+0000 mon.a (mon.0) 1130 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:59.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:59 smithi077 bash[83127]: cluster 2024-09-13T00:47:58.038449+0000 mon.a (mon.0) 1126 : cluster [DBG] osdmap e704: 8 total, 8 up, 8 in 2024-09-13T00:47:59.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:59 smithi077 bash[83127]: audit 2024-09-13T00:47:58.423775+0000 mon.a (mon.0) 1127 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:59.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:59 smithi077 bash[83127]: audit 2024-09-13T00:47:58.425090+0000 mon.a (mon.0) 1128 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:47:59.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:59 smithi077 bash[83127]: audit 2024-09-13T00:47:59.012010+0000 mon.a (mon.0) 1129 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:47:59.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:47:59 smithi077 bash[83127]: audit 2024-09-13T00:47:59.019813+0000 mon.a (mon.0) 1130 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:48:00.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:00 smithi088 bash[68960]: cluster 2024-09-13T00:47:58.637411+0000 mgr.y (mgr.25304) 363 : cluster [DBG] pgmap v687: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 107 KiB/s rd, 11 KiB/s wr, 147 op/s 2024-09-13T00:48:00.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:00 smithi088 bash[68960]: cluster 2024-09-13T00:47:59.040086+0000 mon.a (mon.0) 1131 : cluster [DBG] osdmap e705: 8 total, 8 up, 8 in 2024-09-13T00:48:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:00 smithi088 bash[68960]: audit 2024-09-13T00:47:59.268021+0000 mon.a (mon.0) 1132 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:48:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:00 smithi088 bash[68960]: audit 2024-09-13T00:47:59.278126+0000 mon.a (mon.0) 1133 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:48:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:00 smithi088 bash[68960]: audit 2024-09-13T00:47:59.702875+0000 mon.a (mon.0) 1134 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:48:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:00 smithi088 bash[68960]: audit 2024-09-13T00:47:59.704549+0000 mon.a (mon.0) 1135 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:48:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:00 smithi088 bash[68960]: audit 2024-09-13T00:47:59.714474+0000 mon.a (mon.0) 1136 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:48:00.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:00 smithi077 bash[80681]: cluster 2024-09-13T00:47:58.637411+0000 mgr.y (mgr.25304) 363 : cluster [DBG] pgmap v687: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 107 KiB/s rd, 11 KiB/s wr, 147 op/s 2024-09-13T00:48:00.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:00 smithi077 bash[80681]: cluster 2024-09-13T00:47:59.040086+0000 mon.a (mon.0) 1131 : cluster [DBG] osdmap e705: 8 total, 8 up, 8 in 2024-09-13T00:48:00.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:00 smithi077 bash[80681]: audit 2024-09-13T00:47:59.268021+0000 mon.a (mon.0) 1132 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:48:00.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:00 smithi077 bash[80681]: audit 2024-09-13T00:47:59.278126+0000 mon.a (mon.0) 1133 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:48:00.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:00 smithi077 bash[80681]: audit 2024-09-13T00:47:59.702875+0000 mon.a (mon.0) 1134 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:48:00.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:00 smithi077 bash[80681]: audit 2024-09-13T00:47:59.704549+0000 mon.a (mon.0) 1135 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:48:00.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:00 smithi077 bash[80681]: audit 2024-09-13T00:47:59.714474+0000 mon.a (mon.0) 1136 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:48:00.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:00 smithi077 bash[83127]: cluster 2024-09-13T00:47:58.637411+0000 mgr.y (mgr.25304) 363 : cluster [DBG] pgmap v687: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 107 KiB/s rd, 11 KiB/s wr, 147 op/s 2024-09-13T00:48:00.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:00 smithi077 bash[83127]: cluster 2024-09-13T00:47:59.040086+0000 mon.a (mon.0) 1131 : cluster [DBG] osdmap e705: 8 total, 8 up, 8 in 2024-09-13T00:48:00.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:00 smithi077 bash[83127]: audit 2024-09-13T00:47:59.268021+0000 mon.a (mon.0) 1132 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:48:00.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:00 smithi077 bash[83127]: audit 2024-09-13T00:47:59.278126+0000 mon.a (mon.0) 1133 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:48:00.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:00 smithi077 bash[83127]: audit 2024-09-13T00:47:59.702875+0000 mon.a (mon.0) 1134 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:48:00.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:00 smithi077 bash[83127]: audit 2024-09-13T00:47:59.704549+0000 mon.a (mon.0) 1135 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:48:00.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:00 smithi077 bash[83127]: audit 2024-09-13T00:47:59.714474+0000 mon.a (mon.0) 1136 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:48:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:01 smithi088 bash[68960]: cluster 2024-09-13T00:48:00.056417+0000 mon.a (mon.0) 1137 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2024-09-13T00:48:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:01 smithi088 bash[68960]: audit 2024-09-13T00:48:00.482259+0000 mon.a (mon.0) 1138 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]: dispatch 2024-09-13T00:48:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:01 smithi088 bash[68960]: audit 2024-09-13T00:48:00.482757+0000 mon.a (mon.0) 1139 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1f", "id": [2, 4]}]: dispatch 2024-09-13T00:48:01.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:01 smithi088 bash[68960]: audit 2024-09-13T00:48:01.025872+0000 mon.a (mon.0) 1140 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]': finished 2024-09-13T00:48:01.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:01 smithi088 bash[68960]: audit 2024-09-13T00:48:01.026075+0000 mon.a (mon.0) 1141 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1f", "id": [2, 4]}]': finished 2024-09-13T00:48:01.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:01 smithi088 bash[68960]: cluster 2024-09-13T00:48:01.026197+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2024-09-13T00:48:01.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:01 smithi077 bash[80681]: cluster 2024-09-13T00:48:00.056417+0000 mon.a (mon.0) 1137 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2024-09-13T00:48:01.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:01 smithi077 bash[80681]: audit 2024-09-13T00:48:00.482259+0000 mon.a (mon.0) 1138 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]: dispatch 2024-09-13T00:48:01.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:01 smithi077 bash[80681]: audit 2024-09-13T00:48:00.482757+0000 mon.a (mon.0) 1139 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1f", "id": [2, 4]}]: dispatch 2024-09-13T00:48:01.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:01 smithi077 bash[80681]: audit 2024-09-13T00:48:01.025872+0000 mon.a (mon.0) 1140 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]': finished 2024-09-13T00:48:01.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:01 smithi077 bash[80681]: audit 2024-09-13T00:48:01.026075+0000 mon.a (mon.0) 1141 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1f", "id": [2, 4]}]': finished 2024-09-13T00:48:01.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:01 smithi077 bash[80681]: cluster 2024-09-13T00:48:01.026197+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2024-09-13T00:48:01.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:01 smithi077 bash[83127]: cluster 2024-09-13T00:48:00.056417+0000 mon.a (mon.0) 1137 : cluster [DBG] osdmap e706: 8 total, 8 up, 8 in 2024-09-13T00:48:01.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:01 smithi077 bash[83127]: audit 2024-09-13T00:48:00.482259+0000 mon.a (mon.0) 1138 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]: dispatch 2024-09-13T00:48:01.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:01 smithi077 bash[83127]: audit 2024-09-13T00:48:00.482757+0000 mon.a (mon.0) 1139 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1f", "id": [2, 4]}]: dispatch 2024-09-13T00:48:01.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:01 smithi077 bash[83127]: audit 2024-09-13T00:48:01.025872+0000 mon.a (mon.0) 1140 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.3", "id": [2, 4]}]': finished 2024-09-13T00:48:01.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:01 smithi077 bash[83127]: audit 2024-09-13T00:48:01.026075+0000 mon.a (mon.0) 1141 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1f", "id": [2, 4]}]': finished 2024-09-13T00:48:01.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:01 smithi077 bash[83127]: cluster 2024-09-13T00:48:01.026197+0000 mon.a (mon.0) 1142 : cluster [DBG] osdmap e707: 8 total, 8 up, 8 in 2024-09-13T00:48:02.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:02 smithi088 bash[68960]: cluster 2024-09-13T00:48:00.638455+0000 mgr.y (mgr.25304) 364 : cluster [DBG] pgmap v690: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 227 KiB/s rd, 29 KiB/s wr, 308 op/s 2024-09-13T00:48:02.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:02 smithi088 bash[68960]: cluster 2024-09-13T00:48:02.027348+0000 mon.a (mon.0) 1143 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2024-09-13T00:48:02.351 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:48:02 smithi088 bash[61791]: ts=2024-09-13T00:48:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:48:02.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:02 smithi077 bash[80681]: cluster 2024-09-13T00:48:00.638455+0000 mgr.y (mgr.25304) 364 : cluster [DBG] pgmap v690: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 227 KiB/s rd, 29 KiB/s wr, 308 op/s 2024-09-13T00:48:02.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:02 smithi077 bash[80681]: cluster 2024-09-13T00:48:02.027348+0000 mon.a (mon.0) 1143 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2024-09-13T00:48:02.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:02 smithi077 bash[83127]: cluster 2024-09-13T00:48:00.638455+0000 mgr.y (mgr.25304) 364 : cluster [DBG] pgmap v690: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 227 KiB/s rd, 29 KiB/s wr, 308 op/s 2024-09-13T00:48:02.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:02 smithi077 bash[83127]: cluster 2024-09-13T00:48:02.027348+0000 mon.a (mon.0) 1143 : cluster [DBG] osdmap e708: 8 total, 8 up, 8 in 2024-09-13T00:48:03.104 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.SnapshotLimit (4718 ms) 2024-09-13T00:48:03.104 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.SnapshotLimitPP 2024-09-13T00:48:03.104 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:04.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:04 smithi077 bash[80681]: cluster 2024-09-13T00:48:02.639634+0000 mgr.y (mgr.25304) 365 : cluster [DBG] pgmap v693: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 187 KiB/s rd, 41 KiB/s wr, 235 op/s 2024-09-13T00:48:04.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:04 smithi077 bash[80681]: cluster 2024-09-13T00:48:03.027605+0000 mon.a (mon.0) 1144 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2024-09-13T00:48:04.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:04 smithi077 bash[83127]: cluster 2024-09-13T00:48:02.639634+0000 mgr.y (mgr.25304) 365 : cluster [DBG] pgmap v693: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 187 KiB/s rd, 41 KiB/s wr, 235 op/s 2024-09-13T00:48:04.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:04 smithi077 bash[83127]: cluster 2024-09-13T00:48:03.027605+0000 mon.a (mon.0) 1144 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2024-09-13T00:48:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:04 smithi088 bash[68960]: cluster 2024-09-13T00:48:02.639634+0000 mgr.y (mgr.25304) 365 : cluster [DBG] pgmap v693: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 187 KiB/s rd, 41 KiB/s wr, 235 op/s 2024-09-13T00:48:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:04 smithi088 bash[68960]: cluster 2024-09-13T00:48:03.027605+0000 mon.a (mon.0) 1144 : cluster [DBG] osdmap e709: 8 total, 8 up, 8 in 2024-09-13T00:48:05.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:05 smithi088 bash[68960]: cluster 2024-09-13T00:48:04.092972+0000 mon.a (mon.0) 1145 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2024-09-13T00:48:05.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:05 smithi077 bash[80681]: cluster 2024-09-13T00:48:04.092972+0000 mon.a (mon.0) 1145 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2024-09-13T00:48:05.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:05 smithi077 bash[83127]: cluster 2024-09-13T00:48:04.092972+0000 mon.a (mon.0) 1145 : cluster [DBG] osdmap e710: 8 total, 8 up, 8 in 2024-09-13T00:48:06.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:06 smithi077 bash[80681]: cluster 2024-09-13T00:48:04.642795+0000 mgr.y (mgr.25304) 366 : cluster [DBG] pgmap v696: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 KiB/s rd, 20 KiB/s wr, 68 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:48:06.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:06 smithi077 bash[80681]: cluster 2024-09-13T00:48:05.104031+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2024-09-13T00:48:06.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:06 smithi077 bash[83127]: cluster 2024-09-13T00:48:04.642795+0000 mgr.y (mgr.25304) 366 : cluster [DBG] pgmap v696: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 KiB/s rd, 20 KiB/s wr, 68 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:48:06.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:06 smithi077 bash[83127]: cluster 2024-09-13T00:48:05.104031+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2024-09-13T00:48:06.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:06 smithi088 bash[68960]: cluster 2024-09-13T00:48:04.642795+0000 mgr.y (mgr.25304) 366 : cluster [DBG] pgmap v696: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 KiB/s rd, 20 KiB/s wr, 68 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:48:06.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:06 smithi088 bash[68960]: cluster 2024-09-13T00:48:05.104031+0000 mon.a (mon.0) 1146 : cluster [DBG] osdmap e711: 8 total, 8 up, 8 in 2024-09-13T00:48:07.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:07 smithi077 bash[80681]: cluster 2024-09-13T00:48:06.104736+0000 mon.a (mon.0) 1147 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2024-09-13T00:48:07.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:07 smithi077 bash[83127]: cluster 2024-09-13T00:48:06.104736+0000 mon.a (mon.0) 1147 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2024-09-13T00:48:07.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:07 smithi088 bash[68960]: cluster 2024-09-13T00:48:06.104736+0000 mon.a (mon.0) 1147 : cluster [DBG] osdmap e712: 8 total, 8 up, 8 in 2024-09-13T00:48:08.209 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.SnapshotLimitPP (5105 ms) 2024-09-13T00:48:08.209 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.RebuildObjectMapViaLockOwner 2024-09-13T00:48:08.210 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:08.350 INFO:tasks.workunit.client.0.smithi077.stdout:0.00% done 2024-09-13T00:48:08.351 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.RebuildObjectMapViaLockOwner (142 ms) 2024-09-13T00:48:08.351 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.RenameViaLockOwner 2024-09-13T00:48:08.351 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:08.520 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.RenameViaLockOwner (170 ms) 2024-09-13T00:48:08.521 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.SnapCreateViaLockOwner 2024-09-13T00:48:08.521 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:08.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:08 smithi077 bash[83127]: cluster 2024-09-13T00:48:06.644552+0000 mgr.y (mgr.25304) 367 : cluster [DBG] pgmap v699: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 4.2 KiB/s wr, 62 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:48:08.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:08 smithi077 bash[83127]: cluster 2024-09-13T00:48:07.130860+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2024-09-13T00:48:08.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:08 smithi077 bash[80681]: cluster 2024-09-13T00:48:06.644552+0000 mgr.y (mgr.25304) 367 : cluster [DBG] pgmap v699: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 4.2 KiB/s wr, 62 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:48:08.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:08 smithi077 bash[80681]: cluster 2024-09-13T00:48:07.130860+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2024-09-13T00:48:08.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:08 smithi088 bash[68960]: cluster 2024-09-13T00:48:06.644552+0000 mgr.y (mgr.25304) 367 : cluster [DBG] pgmap v699: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 4.2 KiB/s wr, 62 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:48:08.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:08 smithi088 bash[68960]: cluster 2024-09-13T00:48:07.130860+0000 mon.a (mon.0) 1148 : cluster [DBG] osdmap e713: 8 total, 8 up, 8 in 2024-09-13T00:48:08.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:48:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:48:08] "GET /metrics HTTP/1.1" 200 75158 "" "Prometheus/2.43.0" 2024-09-13T00:48:09.262 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.SnapCreateViaLockOwner (741 ms) 2024-09-13T00:48:09.262 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.SnapRemoveViaLockOwner 2024-09-13T00:48:09.262 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:09.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:09 smithi077 bash[80681]: cluster 2024-09-13T00:48:08.148919+0000 mon.a (mon.0) 1149 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2024-09-13T00:48:09.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:09 smithi077 bash[83127]: cluster 2024-09-13T00:48:08.148919+0000 mon.a (mon.0) 1149 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2024-09-13T00:48:09.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:09 smithi088 bash[68960]: cluster 2024-09-13T00:48:08.148919+0000 mon.a (mon.0) 1149 : cluster [DBG] osdmap e714: 8 total, 8 up, 8 in 2024-09-13T00:48:10.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:10 smithi077 bash[83127]: cluster 2024-09-13T00:48:08.646359+0000 mgr.y (mgr.25304) 368 : cluster [DBG] pgmap v702: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.0 KiB/s wr, 11 op/s 2024-09-13T00:48:10.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:10 smithi077 bash[83127]: cluster 2024-09-13T00:48:09.164128+0000 mon.a (mon.0) 1150 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2024-09-13T00:48:10.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:10 smithi077 bash[80681]: cluster 2024-09-13T00:48:08.646359+0000 mgr.y (mgr.25304) 368 : cluster [DBG] pgmap v702: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.0 KiB/s wr, 11 op/s 2024-09-13T00:48:10.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:10 smithi077 bash[80681]: cluster 2024-09-13T00:48:09.164128+0000 mon.a (mon.0) 1150 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2024-09-13T00:48:10.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:10 smithi088 bash[68960]: cluster 2024-09-13T00:48:08.646359+0000 mgr.y (mgr.25304) 368 : cluster [DBG] pgmap v702: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.5 KiB/s rd, 2.0 KiB/s wr, 11 op/s 2024-09-13T00:48:10.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:10 smithi088 bash[68960]: cluster 2024-09-13T00:48:09.164128+0000 mon.a (mon.0) 1150 : cluster [DBG] osdmap e715: 8 total, 8 up, 8 in 2024-09-13T00:48:11.127 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.SnapRemoveViaLockOwner (1857 ms) 2024-09-13T00:48:11.128 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.UpdateFeaturesViaLockOwner 2024-09-13T00:48:11.128 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:11.255 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.UpdateFeaturesViaLockOwner (136 ms) 2024-09-13T00:48:11.255 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.EnableJournalingViaLockOwner 2024-09-13T00:48:11.255 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:48:11.255 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.EnableJournalingViaLockOwner (0 ms) 2024-09-13T00:48:11.255 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.SnapRemove2 2024-09-13T00:48:11.255 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:11.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:11 smithi077 bash[80681]: cluster 2024-09-13T00:48:10.160982+0000 mon.a (mon.0) 1151 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2024-09-13T00:48:11.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:11 smithi077 bash[80681]: cluster 2024-09-13T00:48:11.027345+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2024-09-13T00:48:11.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:11 smithi077 bash[83127]: cluster 2024-09-13T00:48:10.160982+0000 mon.a (mon.0) 1151 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2024-09-13T00:48:11.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:11 smithi077 bash[83127]: cluster 2024-09-13T00:48:11.027345+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2024-09-13T00:48:11.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:11 smithi088 bash[68960]: cluster 2024-09-13T00:48:10.160982+0000 mon.a (mon.0) 1151 : cluster [DBG] osdmap e716: 8 total, 8 up, 8 in 2024-09-13T00:48:11.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:11 smithi088 bash[68960]: cluster 2024-09-13T00:48:11.027345+0000 mon.a (mon.0) 1152 : cluster [DBG] osdmap e717: 8 total, 8 up, 8 in 2024-09-13T00:48:12.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:12 smithi077 bash[80681]: cluster 2024-09-13T00:48:10.647516+0000 mgr.y (mgr.25304) 369 : cluster [DBG] pgmap v705: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 6.0 KiB/s wr, 92 op/s 2024-09-13T00:48:12.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:12 smithi077 bash[83127]: cluster 2024-09-13T00:48:10.647516+0000 mgr.y (mgr.25304) 369 : cluster [DBG] pgmap v705: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 6.0 KiB/s wr, 92 op/s 2024-09-13T00:48:12.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:12 smithi088 bash[68960]: cluster 2024-09-13T00:48:10.647516+0000 mgr.y (mgr.25304) 369 : cluster [DBG] pgmap v705: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 6.0 KiB/s wr, 92 op/s 2024-09-13T00:48:12.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:48:12 smithi088 bash[61791]: ts=2024-09-13T00:48:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:48:13.267 INFO:tasks.workunit.client.0.smithi077.stdout:100.00% done 2024-09-13T00:48:13.267 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.SnapRemove2 (2012 ms) 2024-09-13T00:48:13.267 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.SnapRenameViaLockOwner 2024-09-13T00:48:13.267 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:13.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:13 smithi077 bash[80681]: cluster 2024-09-13T00:48:12.200992+0000 mon.a (mon.0) 1153 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2024-09-13T00:48:13.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:13 smithi077 bash[83127]: cluster 2024-09-13T00:48:12.200992+0000 mon.a (mon.0) 1153 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2024-09-13T00:48:13.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:13 smithi088 bash[68960]: cluster 2024-09-13T00:48:12.200992+0000 mon.a (mon.0) 1153 : cluster [DBG] osdmap e718: 8 total, 8 up, 8 in 2024-09-13T00:48:14.337 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.SnapRenameViaLockOwner (1070 ms) 2024-09-13T00:48:14.337 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.SnapProtectViaLockOwner 2024-09-13T00:48:14.337 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:14.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:14 smithi077 bash[83127]: cluster 2024-09-13T00:48:12.648604+0000 mgr.y (mgr.25304) 370 : cluster [DBG] pgmap v708: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 78 KiB/s rd, 6.2 KiB/s wr, 100 op/s 2024-09-13T00:48:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:14 smithi077 bash[83127]: cluster 2024-09-13T00:48:13.214399+0000 mon.a (mon.0) 1154 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2024-09-13T00:48:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:14 smithi077 bash[83127]: audit 2024-09-13T00:48:13.421767+0000 mon.a (mon.0) 1155 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:14 smithi077 bash[80681]: cluster 2024-09-13T00:48:12.648604+0000 mgr.y (mgr.25304) 370 : cluster [DBG] pgmap v708: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 78 KiB/s rd, 6.2 KiB/s wr, 100 op/s 2024-09-13T00:48:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:14 smithi077 bash[80681]: cluster 2024-09-13T00:48:13.214399+0000 mon.a (mon.0) 1154 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2024-09-13T00:48:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:14 smithi077 bash[80681]: audit 2024-09-13T00:48:13.421767+0000 mon.a (mon.0) 1155 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:14.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:14 smithi088 bash[68960]: cluster 2024-09-13T00:48:12.648604+0000 mgr.y (mgr.25304) 370 : cluster [DBG] pgmap v708: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 78 KiB/s rd, 6.2 KiB/s wr, 100 op/s 2024-09-13T00:48:14.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:14 smithi088 bash[68960]: cluster 2024-09-13T00:48:13.214399+0000 mon.a (mon.0) 1154 : cluster [DBG] osdmap e719: 8 total, 8 up, 8 in 2024-09-13T00:48:14.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:14 smithi088 bash[68960]: audit 2024-09-13T00:48:13.421767+0000 mon.a (mon.0) 1155 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:15.355 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.SnapProtectViaLockOwner (1018 ms) 2024-09-13T00:48:15.355 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.SnapUnprotectViaLockOwner 2024-09-13T00:48:15.355 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:15.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:15 smithi077 bash[80681]: cluster 2024-09-13T00:48:14.220661+0000 mon.a (mon.0) 1156 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2024-09-13T00:48:15.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:15 smithi077 bash[83127]: cluster 2024-09-13T00:48:14.220661+0000 mon.a (mon.0) 1156 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2024-09-13T00:48:15.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:15 smithi088 bash[68960]: cluster 2024-09-13T00:48:14.220661+0000 mon.a (mon.0) 1156 : cluster [DBG] osdmap e720: 8 total, 8 up, 8 in 2024-09-13T00:48:16.178 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.SnapUnprotectViaLockOwner (823 ms) 2024-09-13T00:48:16.178 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.FlattenViaLockOwner 2024-09-13T00:48:16.178 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:16.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:16 smithi077 bash[80681]: cluster 2024-09-13T00:48:14.651309+0000 mgr.y (mgr.25304) 371 : cluster [DBG] pgmap v711: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 158 KiB/s rd, 14 KiB/s wr, 217 op/s 2024-09-13T00:48:16.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:16 smithi077 bash[80681]: cluster 2024-09-13T00:48:15.233054+0000 mon.a (mon.0) 1157 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2024-09-13T00:48:16.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:16 smithi077 bash[80681]: cluster 2024-09-13T00:48:16.029633+0000 mon.a (mon.0) 1158 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2024-09-13T00:48:16.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:16 smithi077 bash[83127]: cluster 2024-09-13T00:48:14.651309+0000 mgr.y (mgr.25304) 371 : cluster [DBG] pgmap v711: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 158 KiB/s rd, 14 KiB/s wr, 217 op/s 2024-09-13T00:48:16.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:16 smithi077 bash[83127]: cluster 2024-09-13T00:48:15.233054+0000 mon.a (mon.0) 1157 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2024-09-13T00:48:16.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:16 smithi077 bash[83127]: cluster 2024-09-13T00:48:16.029633+0000 mon.a (mon.0) 1158 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2024-09-13T00:48:16.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:16 smithi088 bash[68960]: cluster 2024-09-13T00:48:14.651309+0000 mgr.y (mgr.25304) 371 : cluster [DBG] pgmap v711: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 158 KiB/s rd, 14 KiB/s wr, 217 op/s 2024-09-13T00:48:16.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:16 smithi088 bash[68960]: cluster 2024-09-13T00:48:15.233054+0000 mon.a (mon.0) 1157 : cluster [DBG] osdmap e721: 8 total, 8 up, 8 in 2024-09-13T00:48:16.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:16 smithi088 bash[68960]: cluster 2024-09-13T00:48:16.029633+0000 mon.a (mon.0) 1158 : cluster [DBG] osdmap e722: 8 total, 8 up, 8 in 2024-09-13T00:48:17.517 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.FlattenViaLockOwner (1340 ms) 2024-09-13T00:48:17.518 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ResizeViaLockOwner 2024-09-13T00:48:17.518 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:17.621 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ResizeViaLockOwner (103 ms) 2024-09-13T00:48:17.621 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.SparsifyViaLockOwner 2024-09-13T00:48:17.621 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:17.722 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.SparsifyViaLockOwner (102 ms) 2024-09-13T00:48:17.722 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ObjectMapConsistentSnap 2024-09-13T00:48:17.723 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:18.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:18 smithi077 bash[80681]: cluster 2024-09-13T00:48:16.653102+0000 mgr.y (mgr.25304) 372 : cluster [DBG] pgmap v714: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 14 KiB/s wr, 242 op/s 2024-09-13T00:48:18.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:18 smithi077 bash[80681]: cluster 2024-09-13T00:48:17.244738+0000 mon.a (mon.0) 1159 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2024-09-13T00:48:18.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:18 smithi077 bash[83127]: cluster 2024-09-13T00:48:16.653102+0000 mgr.y (mgr.25304) 372 : cluster [DBG] pgmap v714: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 14 KiB/s wr, 242 op/s 2024-09-13T00:48:18.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:18 smithi077 bash[83127]: cluster 2024-09-13T00:48:17.244738+0000 mon.a (mon.0) 1159 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2024-09-13T00:48:18.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:18 smithi088 bash[68960]: cluster 2024-09-13T00:48:16.653102+0000 mgr.y (mgr.25304) 372 : cluster [DBG] pgmap v714: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 14 KiB/s wr, 242 op/s 2024-09-13T00:48:18.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:18 smithi088 bash[68960]: cluster 2024-09-13T00:48:17.244738+0000 mon.a (mon.0) 1159 : cluster [DBG] osdmap e723: 8 total, 8 up, 8 in 2024-09-13T00:48:18.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:48:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:48:18] "GET /metrics HTTP/1.1" 200 75157 "" "Prometheus/2.43.0" 2024-09-13T00:48:19.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:19 smithi077 bash[80681]: cluster 2024-09-13T00:48:18.254419+0000 mon.a (mon.0) 1160 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2024-09-13T00:48:19.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:19 smithi077 bash[83127]: cluster 2024-09-13T00:48:18.254419+0000 mon.a (mon.0) 1160 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2024-09-13T00:48:19.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:19 smithi088 bash[68960]: cluster 2024-09-13T00:48:18.254419+0000 mon.a (mon.0) 1160 : cluster [DBG] osdmap e724: 8 total, 8 up, 8 in 2024-09-13T00:48:20.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:20 smithi077 bash[80681]: cluster 2024-09-13T00:48:18.654840+0000 mgr.y (mgr.25304) 373 : cluster [DBG] pgmap v717: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 4.5 KiB/s wr, 97 op/s 2024-09-13T00:48:20.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:20 smithi077 bash[80681]: cluster 2024-09-13T00:48:19.267716+0000 mon.a (mon.0) 1161 : cluster [DBG] osdmap e725: 8 total, 8 up, 8 in 2024-09-13T00:48:20.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:20 smithi077 bash[83127]: cluster 2024-09-13T00:48:18.654840+0000 mgr.y (mgr.25304) 373 : cluster [DBG] pgmap v717: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 4.5 KiB/s wr, 97 op/s 2024-09-13T00:48:20.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:20 smithi077 bash[83127]: cluster 2024-09-13T00:48:19.267716+0000 mon.a (mon.0) 1161 : cluster [DBG] osdmap e725: 8 total, 8 up, 8 in 2024-09-13T00:48:20.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:20 smithi088 bash[68960]: cluster 2024-09-13T00:48:18.654840+0000 mgr.y (mgr.25304) 373 : cluster [DBG] pgmap v717: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 4.5 KiB/s wr, 97 op/s 2024-09-13T00:48:20.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:20 smithi088 bash[68960]: cluster 2024-09-13T00:48:19.267716+0000 mon.a (mon.0) 1161 : cluster [DBG] osdmap e725: 8 total, 8 up, 8 in 2024-09-13T00:48:21.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:21 smithi088 bash[68960]: cluster 2024-09-13T00:48:20.269977+0000 mon.a (mon.0) 1162 : cluster [DBG] osdmap e726: 8 total, 8 up, 8 in 2024-09-13T00:48:21.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:21 smithi088 bash[68960]: cluster 2024-09-13T00:48:21.036013+0000 mon.a (mon.0) 1163 : cluster [DBG] osdmap e727: 8 total, 8 up, 8 in 2024-09-13T00:48:21.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:21 smithi077 bash[80681]: cluster 2024-09-13T00:48:20.269977+0000 mon.a (mon.0) 1162 : cluster [DBG] osdmap e726: 8 total, 8 up, 8 in 2024-09-13T00:48:21.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:21 smithi077 bash[80681]: cluster 2024-09-13T00:48:21.036013+0000 mon.a (mon.0) 1163 : cluster [DBG] osdmap e727: 8 total, 8 up, 8 in 2024-09-13T00:48:21.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:21 smithi077 bash[83127]: cluster 2024-09-13T00:48:20.269977+0000 mon.a (mon.0) 1162 : cluster [DBG] osdmap e726: 8 total, 8 up, 8 in 2024-09-13T00:48:21.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:21 smithi077 bash[83127]: cluster 2024-09-13T00:48:21.036013+0000 mon.a (mon.0) 1163 : cluster [DBG] osdmap e727: 8 total, 8 up, 8 in 2024-09-13T00:48:22.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:48:22 smithi088 bash[61791]: ts=2024-09-13T00:48:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:48:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:22 smithi088 bash[68960]: cluster 2024-09-13T00:48:20.655901+0000 mgr.y (mgr.25304) 374 : cluster [DBG] pgmap v720: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 165 KiB/s rd, 11 KiB/s wr, 221 op/s 2024-09-13T00:48:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:22 smithi088 bash[68960]: cluster 2024-09-13T00:48:22.036365+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e728: 8 total, 8 up, 8 in 2024-09-13T00:48:22.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:22 smithi077 bash[80681]: cluster 2024-09-13T00:48:20.655901+0000 mgr.y (mgr.25304) 374 : cluster [DBG] pgmap v720: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 165 KiB/s rd, 11 KiB/s wr, 221 op/s 2024-09-13T00:48:22.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:22 smithi077 bash[80681]: cluster 2024-09-13T00:48:22.036365+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e728: 8 total, 8 up, 8 in 2024-09-13T00:48:22.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:22 smithi077 bash[83127]: cluster 2024-09-13T00:48:20.655901+0000 mgr.y (mgr.25304) 374 : cluster [DBG] pgmap v720: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 165 KiB/s rd, 11 KiB/s wr, 221 op/s 2024-09-13T00:48:22.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:22 smithi077 bash[83127]: cluster 2024-09-13T00:48:22.036365+0000 mon.a (mon.0) 1164 : cluster [DBG] osdmap e728: 8 total, 8 up, 8 in 2024-09-13T00:48:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:24 smithi088 bash[68960]: cluster 2024-09-13T00:48:22.656989+0000 mgr.y (mgr.25304) 375 : cluster [DBG] pgmap v723: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 164 KiB/s rd, 10 KiB/s wr, 216 op/s 2024-09-13T00:48:24.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:24 smithi088 bash[68960]: cluster 2024-09-13T00:48:23.038930+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e729: 8 total, 8 up, 8 in 2024-09-13T00:48:24.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:24 smithi077 bash[80681]: cluster 2024-09-13T00:48:22.656989+0000 mgr.y (mgr.25304) 375 : cluster [DBG] pgmap v723: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 164 KiB/s rd, 10 KiB/s wr, 216 op/s 2024-09-13T00:48:24.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:24 smithi077 bash[80681]: cluster 2024-09-13T00:48:23.038930+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e729: 8 total, 8 up, 8 in 2024-09-13T00:48:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:24 smithi077 bash[83127]: cluster 2024-09-13T00:48:22.656989+0000 mgr.y (mgr.25304) 375 : cluster [DBG] pgmap v723: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 164 KiB/s rd, 10 KiB/s wr, 216 op/s 2024-09-13T00:48:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:24 smithi077 bash[83127]: cluster 2024-09-13T00:48:23.038930+0000 mon.a (mon.0) 1165 : cluster [DBG] osdmap e729: 8 total, 8 up, 8 in 2024-09-13T00:48:25.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:25 smithi088 bash[68960]: cluster 2024-09-13T00:48:24.051340+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e730: 8 total, 8 up, 8 in 2024-09-13T00:48:25.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:25 smithi077 bash[80681]: cluster 2024-09-13T00:48:24.051340+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e730: 8 total, 8 up, 8 in 2024-09-13T00:48:25.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:25 smithi077 bash[83127]: cluster 2024-09-13T00:48:24.051340+0000 mon.a (mon.0) 1166 : cluster [DBG] osdmap e730: 8 total, 8 up, 8 in 2024-09-13T00:48:26.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:26 smithi088 bash[68960]: cluster 2024-09-13T00:48:24.660263+0000 mgr.y (mgr.25304) 376 : cluster [DBG] pgmap v726: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 2.2 KiB/s wr, 55 op/s 2024-09-13T00:48:26.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:26 smithi088 bash[68960]: cluster 2024-09-13T00:48:25.062012+0000 mon.a (mon.0) 1167 : cluster [DBG] osdmap e731: 8 total, 8 up, 8 in 2024-09-13T00:48:26.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:26 smithi077 bash[80681]: cluster 2024-09-13T00:48:24.660263+0000 mgr.y (mgr.25304) 376 : cluster [DBG] pgmap v726: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 2.2 KiB/s wr, 55 op/s 2024-09-13T00:48:26.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:26 smithi077 bash[80681]: cluster 2024-09-13T00:48:25.062012+0000 mon.a (mon.0) 1167 : cluster [DBG] osdmap e731: 8 total, 8 up, 8 in 2024-09-13T00:48:26.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:26 smithi077 bash[83127]: cluster 2024-09-13T00:48:24.660263+0000 mgr.y (mgr.25304) 376 : cluster [DBG] pgmap v726: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 2.2 KiB/s wr, 55 op/s 2024-09-13T00:48:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:26 smithi077 bash[83127]: cluster 2024-09-13T00:48:25.062012+0000 mon.a (mon.0) 1167 : cluster [DBG] osdmap e731: 8 total, 8 up, 8 in 2024-09-13T00:48:27.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:27 smithi088 bash[68960]: cluster 2024-09-13T00:48:26.071337+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e732: 8 total, 8 up, 8 in 2024-09-13T00:48:27.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:27 smithi077 bash[80681]: cluster 2024-09-13T00:48:26.071337+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e732: 8 total, 8 up, 8 in 2024-09-13T00:48:27.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:27 smithi077 bash[83127]: cluster 2024-09-13T00:48:26.071337+0000 mon.a (mon.0) 1168 : cluster [DBG] osdmap e732: 8 total, 8 up, 8 in 2024-09-13T00:48:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:28 smithi088 bash[68960]: cluster 2024-09-13T00:48:26.662021+0000 mgr.y (mgr.25304) 377 : cluster [DBG] pgmap v729: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 61 KiB/s rd, 3.7 KiB/s wr, 81 op/s 2024-09-13T00:48:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:28 smithi088 bash[68960]: cluster 2024-09-13T00:48:27.084658+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e733: 8 total, 8 up, 8 in 2024-09-13T00:48:28.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:28 smithi077 bash[80681]: cluster 2024-09-13T00:48:26.662021+0000 mgr.y (mgr.25304) 377 : cluster [DBG] pgmap v729: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 61 KiB/s rd, 3.7 KiB/s wr, 81 op/s 2024-09-13T00:48:28.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:28 smithi077 bash[80681]: cluster 2024-09-13T00:48:27.084658+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e733: 8 total, 8 up, 8 in 2024-09-13T00:48:28.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:28 smithi077 bash[83127]: cluster 2024-09-13T00:48:26.662021+0000 mgr.y (mgr.25304) 377 : cluster [DBG] pgmap v729: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 61 KiB/s rd, 3.7 KiB/s wr, 81 op/s 2024-09-13T00:48:28.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:28 smithi077 bash[83127]: cluster 2024-09-13T00:48:27.084658+0000 mon.a (mon.0) 1169 : cluster [DBG] osdmap e733: 8 total, 8 up, 8 in 2024-09-13T00:48:28.632 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ObjectMapConsistentSnap (10909 ms) 2024-09-13T00:48:28.632 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.Metadata 2024-09-13T00:48:28.633 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:28.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:48:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:48:28] "GET /metrics HTTP/1.1" 200 75150 "" "Prometheus/2.43.0" 2024-09-13T00:48:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:29 smithi088 bash[68960]: audit 2024-09-13T00:48:28.422162+0000 mon.a (mon.0) 1170 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:29.385 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.Metadata (753 ms) 2024-09-13T00:48:29.385 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.MetadataPP 2024-09-13T00:48:29.386 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:29.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:29 smithi077 bash[80681]: audit 2024-09-13T00:48:28.422162+0000 mon.a (mon.0) 1170 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:29.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:29 smithi077 bash[83127]: audit 2024-09-13T00:48:28.422162+0000 mon.a (mon.0) 1170 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:30.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:30 smithi088 bash[68960]: cluster 2024-09-13T00:48:28.665068+0000 mgr.y (mgr.25304) 378 : cluster [DBG] pgmap v731: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 61 KiB/s wr, 276 op/s 2024-09-13T00:48:30.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:30 smithi088 bash[68960]: cluster 2024-09-13T00:48:29.094279+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e734: 8 total, 8 up, 8 in 2024-09-13T00:48:30.377 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.MetadataPP (991 ms) 2024-09-13T00:48:30.377 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.UpdateFeatures 2024-09-13T00:48:30.377 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:30.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:30 smithi077 bash[83127]: cluster 2024-09-13T00:48:28.665068+0000 mgr.y (mgr.25304) 378 : cluster [DBG] pgmap v731: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 61 KiB/s wr, 276 op/s 2024-09-13T00:48:30.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:30 smithi077 bash[83127]: cluster 2024-09-13T00:48:29.094279+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e734: 8 total, 8 up, 8 in 2024-09-13T00:48:30.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:30 smithi077 bash[80681]: cluster 2024-09-13T00:48:28.665068+0000 mgr.y (mgr.25304) 378 : cluster [DBG] pgmap v731: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 61 KiB/s wr, 276 op/s 2024-09-13T00:48:30.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:30 smithi077 bash[80681]: cluster 2024-09-13T00:48:29.094279+0000 mon.a (mon.0) 1171 : cluster [DBG] osdmap e734: 8 total, 8 up, 8 in 2024-09-13T00:48:30.655 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.UpdateFeatures (278 ms) 2024-09-13T00:48:30.656 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.FeaturesBitmaskString 2024-09-13T00:48:30.656 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.FeaturesBitmaskString (1 ms) 2024-09-13T00:48:30.656 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.RebuildObjectMap 2024-09-13T00:48:30.656 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:31.258 INFO:tasks.workunit.client.0.smithi077.stdout:0.00% done 2024-09-13T00:48:31.258 INFO:tasks.workunit.client.0.smithi077.stdout:25.00% done 2024-09-13T00:48:31.258 INFO:tasks.workunit.client.0.smithi077.stdout:50.00% done 2024-09-13T00:48:31.258 INFO:tasks.workunit.client.0.smithi077.stdout:75.00% done 2024-09-13T00:48:31.258 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.RebuildObjectMap (602 ms) 2024-09-13T00:48:31.258 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.RebuildNewObjectMap 2024-09-13T00:48:31.344 INFO:tasks.workunit.client.0.smithi077.stdout:0.00% done 2024-09-13T00:48:31.345 INFO:tasks.workunit.client.0.smithi077.stdout:25.00% done 2024-09-13T00:48:31.345 INFO:tasks.workunit.client.0.smithi077.stdout:50.00% done 2024-09-13T00:48:31.345 INFO:tasks.workunit.client.0.smithi077.stdout:75.00% done 2024-09-13T00:48:31.345 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.RebuildNewObjectMap (87 ms) 2024-09-13T00:48:31.345 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.CheckObjectMap 2024-09-13T00:48:31.345 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:31.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:31 smithi077 bash[80681]: cluster 2024-09-13T00:48:30.105993+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e735: 8 total, 8 up, 8 in 2024-09-13T00:48:31.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:31 smithi077 bash[83127]: cluster 2024-09-13T00:48:30.105993+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e735: 8 total, 8 up, 8 in 2024-09-13T00:48:31.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:31 smithi088 bash[68960]: cluster 2024-09-13T00:48:30.105993+0000 mon.a (mon.0) 1172 : cluster [DBG] osdmap e735: 8 total, 8 up, 8 in 2024-09-13T00:48:32.287 INFO:tasks.workunit.client.0.smithi077.stdout:0.00% done 2024-09-13T00:48:32.288 INFO:tasks.workunit.client.0.smithi077.stdout:25.00% done 2024-09-13T00:48:32.288 INFO:tasks.workunit.client.0.smithi077.stdout:50.00% done 2024-09-13T00:48:32.288 INFO:tasks.workunit.client.0.smithi077.stdout:75.00% done 2024-09-13T00:48:32.288 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.CheckObjectMap (942 ms) 2024-09-13T00:48:32.288 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.BlockingAIO 2024-09-13T00:48:32.288 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:32.359 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.BlockingAIO (72 ms) 2024-09-13T00:48:32.359 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ExclusiveLockTransition 2024-09-13T00:48:32.359 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:32.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:32 smithi077 bash[80681]: cluster 2024-09-13T00:48:30.666158+0000 mgr.y (mgr.25304) 379 : cluster [DBG] pgmap v734: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 251 KiB/s rd, 97 KiB/s wr, 407 op/s 2024-09-13T00:48:32.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:32 smithi077 bash[80681]: cluster 2024-09-13T00:48:31.116592+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e736: 8 total, 8 up, 8 in 2024-09-13T00:48:32.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:32 smithi077 bash[83127]: cluster 2024-09-13T00:48:30.666158+0000 mgr.y (mgr.25304) 379 : cluster [DBG] pgmap v734: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 251 KiB/s rd, 97 KiB/s wr, 407 op/s 2024-09-13T00:48:32.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:32 smithi077 bash[83127]: cluster 2024-09-13T00:48:31.116592+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e736: 8 total, 8 up, 8 in 2024-09-13T00:48:32.597 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ExclusiveLockTransition (238 ms) 2024-09-13T00:48:32.597 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ExclusiveLockReadTransition 2024-09-13T00:48:32.597 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:48:32.597 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ExclusiveLockReadTransition (0 ms) 2024-09-13T00:48:32.597 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.CacheMayCopyOnWrite 2024-09-13T00:48:32.597 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:32.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:48:32 smithi088 bash[61791]: ts=2024-09-13T00:48:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:48:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:32 smithi088 bash[68960]: cluster 2024-09-13T00:48:30.666158+0000 mgr.y (mgr.25304) 379 : cluster [DBG] pgmap v734: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 251 KiB/s rd, 97 KiB/s wr, 407 op/s 2024-09-13T00:48:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:32 smithi088 bash[68960]: cluster 2024-09-13T00:48:31.116592+0000 mon.a (mon.0) 1173 : cluster [DBG] osdmap e736: 8 total, 8 up, 8 in 2024-09-13T00:48:33.349 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.CacheMayCopyOnWrite (752 ms) 2024-09-13T00:48:33.349 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot 2024-09-13T00:48:33.349 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:33.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:33 smithi077 bash[80681]: cluster 2024-09-13T00:48:32.119151+0000 mon.a (mon.0) 1174 : cluster [DBG] osdmap e737: 8 total, 8 up, 8 in 2024-09-13T00:48:33.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:33 smithi077 bash[83127]: cluster 2024-09-13T00:48:32.119151+0000 mon.a (mon.0) 1174 : cluster [DBG] osdmap e737: 8 total, 8 up, 8 in 2024-09-13T00:48:33.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:33 smithi088 bash[68960]: cluster 2024-09-13T00:48:32.119151+0000 mon.a (mon.0) 1174 : cluster [DBG] osdmap e737: 8 total, 8 up, 8 in 2024-09-13T00:48:34.235 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot (886 ms) 2024-09-13T00:48:34.235 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestImageOptions 2024-09-13T00:48:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:34 smithi077 bash[80681]: cluster 2024-09-13T00:48:32.667415+0000 mgr.y (mgr.25304) 380 : cluster [DBG] pgmap v737: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 144 KiB/s rd, 56 KiB/s wr, 240 op/s 2024-09-13T00:48:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:34 smithi077 bash[80681]: cluster 2024-09-13T00:48:33.149211+0000 mon.a (mon.0) 1175 : cluster [DBG] osdmap e738: 8 total, 8 up, 8 in 2024-09-13T00:48:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:34 smithi077 bash[83127]: cluster 2024-09-13T00:48:32.667415+0000 mgr.y (mgr.25304) 380 : cluster [DBG] pgmap v737: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 144 KiB/s rd, 56 KiB/s wr, 240 op/s 2024-09-13T00:48:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:34 smithi077 bash[83127]: cluster 2024-09-13T00:48:33.149211+0000 mon.a (mon.0) 1175 : cluster [DBG] osdmap e738: 8 total, 8 up, 8 in 2024-09-13T00:48:34.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:34 smithi088 bash[68960]: cluster 2024-09-13T00:48:32.667415+0000 mgr.y (mgr.25304) 380 : cluster [DBG] pgmap v737: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 144 KiB/s rd, 56 KiB/s wr, 240 op/s 2024-09-13T00:48:34.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:34 smithi088 bash[68960]: cluster 2024-09-13T00:48:33.149211+0000 mon.a (mon.0) 1175 : cluster [DBG] osdmap e738: 8 total, 8 up, 8 in 2024-09-13T00:48:35.466 INFO:tasks.workunit.client.0.smithi077.stdout:0.00% done 2024-09-13T00:48:35.467 INFO:tasks.workunit.client.0.smithi077.stdout:100.00% done 2024-09-13T00:48:35.467 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestImageOptions (1232 ms) 2024-09-13T00:48:35.467 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestImageOptionsPP 2024-09-13T00:48:35.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:35 smithi077 bash[80681]: cluster 2024-09-13T00:48:34.160941+0000 mon.a (mon.0) 1176 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2024-09-13T00:48:35.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:35 smithi077 bash[83127]: cluster 2024-09-13T00:48:34.160941+0000 mon.a (mon.0) 1176 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2024-09-13T00:48:35.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:35 smithi088 bash[68960]: cluster 2024-09-13T00:48:34.160941+0000 mon.a (mon.0) 1176 : cluster [DBG] osdmap e739: 8 total, 8 up, 8 in 2024-09-13T00:48:36.433 INFO:tasks.workunit.client.0.smithi077.stdout:0.00% done 2024-09-13T00:48:36.433 INFO:tasks.workunit.client.0.smithi077.stdout:100.00% done 2024-09-13T00:48:36.433 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestImageOptionsPP (966 ms) 2024-09-13T00:48:36.433 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.EventSocketPipe 2024-09-13T00:48:36.433 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.EventSocketPipe (0 ms) 2024-09-13T00:48:36.433 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.EventSocketEventfd 2024-09-13T00:48:36.433 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.EventSocketEventfd (0 ms) 2024-09-13T00:48:36.433 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ImagePollIO 2024-09-13T00:48:36.433 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:36.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:36 smithi077 bash[83127]: cluster 2024-09-13T00:48:34.670997+0000 mgr.y (mgr.25304) 381 : cluster [DBG] pgmap v740: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 667 KiB/s rd, 209 KiB/s wr, 944 op/s 2024-09-13T00:48:36.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:36 smithi077 bash[83127]: cluster 2024-09-13T00:48:35.169895+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2024-09-13T00:48:36.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:36 smithi077 bash[80681]: cluster 2024-09-13T00:48:34.670997+0000 mgr.y (mgr.25304) 381 : cluster [DBG] pgmap v740: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 667 KiB/s rd, 209 KiB/s wr, 944 op/s 2024-09-13T00:48:36.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:36 smithi077 bash[80681]: cluster 2024-09-13T00:48:35.169895+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2024-09-13T00:48:36.530 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:48:36.530 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:48:36.531 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:48:36.531 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:48:36.531 INFO:tasks.workunit.client.0.smithi077.stdout:wrote: 512 2024-09-13T00:48:36.531 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:48:36.531 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:48:36.531 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:48:36.531 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:48:36.531 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:48:36.531 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:48:36.531 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:48:36.531 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:48:36.531 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:48:36.531 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:48:36.532 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:48:36.532 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:48:36.532 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:48:36.532 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:48:36.532 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:48:36.532 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:48:36.532 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:48:36.532 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:48:36.532 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:48:36.532 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:48:36.532 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:48:36.532 INFO:tasks.workunit.client.0.smithi077.stdout:started write 2024-09-13T00:48:36.533 INFO:tasks.workunit.client.0.smithi077.stdout:write completion cb called! 2024-09-13T00:48:36.533 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 0 2024-09-13T00:48:36.533 INFO:tasks.workunit.client.0.smithi077.stdout:finished write 2024-09-13T00:48:36.533 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:48:36.533 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:48:36.533 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:48:36.533 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:48:36.533 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:48:36.533 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:48:36.533 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:48:36.533 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:48:36.533 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:48:36.533 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:48:36.533 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:48:36.534 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:48:36.534 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:48:36.534 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:48:36.534 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:48:36.534 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:48:36.534 INFO:tasks.workunit.client.0.smithi077.stdout:created completion 2024-09-13T00:48:36.534 INFO:tasks.workunit.client.0.smithi077.stdout:started read 2024-09-13T00:48:36.534 INFO:tasks.workunit.client.0.smithi077.stdout:read completion cb called! 2024-09-13T00:48:36.534 INFO:tasks.workunit.client.0.smithi077.stdout:return value is: 512 2024-09-13T00:48:36.534 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ImagePollIO (98 ms) 2024-09-13T00:48:36.534 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.Mirror 2024-09-13T00:48:36.541 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:36.562 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:36.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:36 smithi088 bash[68960]: cluster 2024-09-13T00:48:34.670997+0000 mgr.y (mgr.25304) 381 : cluster [DBG] pgmap v740: 617 pgs: 617 active+clean; 139 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 667 KiB/s rd, 209 KiB/s wr, 944 op/s 2024-09-13T00:48:36.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:36 smithi088 bash[68960]: cluster 2024-09-13T00:48:35.169895+0000 mon.a (mon.0) 1177 : cluster [DBG] osdmap e740: 8 total, 8 up, 8 in 2024-09-13T00:48:37.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:37 smithi077 bash[83127]: cluster 2024-09-13T00:48:36.182854+0000 mon.a (mon.0) 1178 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2024-09-13T00:48:37.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:37 smithi077 bash[80681]: cluster 2024-09-13T00:48:36.182854+0000 mon.a (mon.0) 1178 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2024-09-13T00:48:37.580 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.Mirror (1049 ms) 2024-09-13T00:48:37.580 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.MirrorPeerAttributes 2024-09-13T00:48:37.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:37 smithi088 bash[68960]: cluster 2024-09-13T00:48:36.182854+0000 mon.a (mon.0) 1178 : cluster [DBG] osdmap e741: 8 total, 8 up, 8 in 2024-09-13T00:48:37.616 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.MirrorPeerAttributes (36 ms) 2024-09-13T00:48:37.616 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.CreateWithMirrorEnabled 2024-09-13T00:48:38.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[80681]: cluster 2024-09-13T00:48:36.672768+0000 mgr.y (mgr.25304) 382 : cluster [DBG] pgmap v743: 617 pgs: 617 active+clean; 141 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 695 KiB/s rd, 593 KiB/s wr, 965 op/s 2024-09-13T00:48:38.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[80681]: cluster 2024-09-13T00:48:37.194152+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2024-09-13T00:48:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[80681]: audit 2024-09-13T00:48:37.500794+0000 mon.a (mon.0) 1180 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]: dispatch 2024-09-13T00:48:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[80681]: audit 2024-09-13T00:48:37.509805+0000 mon.a (mon.0) 1181 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]': finished 2024-09-13T00:48:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[80681]: audit 2024-09-13T00:48:37.523331+0000 mon.a (mon.0) 1182 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/e9db9f7d-1382-4c7b-a78e-58638a4a2d88"}]: dispatch 2024-09-13T00:48:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[80681]: audit 2024-09-13T00:48:37.533928+0000 mon.a (mon.0) 1183 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/e9db9f7d-1382-4c7b-a78e-58638a4a2d88"}]': finished 2024-09-13T00:48:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[80681]: audit 2024-09-13T00:48:37.552736+0000 mon.a (mon.0) 1184 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b594d1c7-285b-408a-b469-a8acf1e8fdb7"}]: dispatch 2024-09-13T00:48:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[80681]: audit 2024-09-13T00:48:37.558976+0000 mon.a (mon.0) 1185 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b594d1c7-285b-408a-b469-a8acf1e8fdb7"}]': finished 2024-09-13T00:48:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[80681]: audit 2024-09-13T00:48:37.563813+0000 mon.a (mon.0) 1186 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/4d2838e7-a94f-4b83-b450-6cadbe20bf2e"}]: dispatch 2024-09-13T00:48:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[80681]: audit 2024-09-13T00:48:37.571865+0000 mon.a (mon.0) 1187 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/4d2838e7-a94f-4b83-b450-6cadbe20bf2e"}]': finished 2024-09-13T00:48:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[80681]: audit 2024-09-13T00:48:37.591823+0000 mon.a (mon.0) 1188 : audit [DBG] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/18/6f621f21-009f-477f-9526-7f8242e7000c"}]: dispatch 2024-09-13T00:48:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[80681]: audit 2024-09-13T00:48:37.600691+0000 mon.a (mon.0) 1189 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' 2024-09-13T00:48:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[80681]: audit 2024-09-13T00:48:37.601467+0000 mon.a (mon.0) 1190 : audit [DBG] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/18/6f621f21-009f-477f-9526-7f8242e7000c"}]: dispatch 2024-09-13T00:48:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[80681]: audit 2024-09-13T00:48:37.602256+0000 mon.a (mon.0) 1191 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/6f621f21-009f-477f-9526-7f8242e7000c"}]: dispatch 2024-09-13T00:48:38.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[80681]: audit 2024-09-13T00:48:37.609916+0000 mon.a (mon.0) 1192 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/6f621f21-009f-477f-9526-7f8242e7000c"}]': finished 2024-09-13T00:48:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[83127]: cluster 2024-09-13T00:48:36.672768+0000 mgr.y (mgr.25304) 382 : cluster [DBG] pgmap v743: 617 pgs: 617 active+clean; 141 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 695 KiB/s rd, 593 KiB/s wr, 965 op/s 2024-09-13T00:48:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[83127]: cluster 2024-09-13T00:48:37.194152+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2024-09-13T00:48:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[83127]: audit 2024-09-13T00:48:37.500794+0000 mon.a (mon.0) 1180 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]: dispatch 2024-09-13T00:48:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[83127]: audit 2024-09-13T00:48:37.509805+0000 mon.a (mon.0) 1181 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]': finished 2024-09-13T00:48:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[83127]: audit 2024-09-13T00:48:37.523331+0000 mon.a (mon.0) 1182 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/e9db9f7d-1382-4c7b-a78e-58638a4a2d88"}]: dispatch 2024-09-13T00:48:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[83127]: audit 2024-09-13T00:48:37.533928+0000 mon.a (mon.0) 1183 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/e9db9f7d-1382-4c7b-a78e-58638a4a2d88"}]': finished 2024-09-13T00:48:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[83127]: audit 2024-09-13T00:48:37.552736+0000 mon.a (mon.0) 1184 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b594d1c7-285b-408a-b469-a8acf1e8fdb7"}]: dispatch 2024-09-13T00:48:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[83127]: audit 2024-09-13T00:48:37.558976+0000 mon.a (mon.0) 1185 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b594d1c7-285b-408a-b469-a8acf1e8fdb7"}]': finished 2024-09-13T00:48:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[83127]: audit 2024-09-13T00:48:37.563813+0000 mon.a (mon.0) 1186 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/4d2838e7-a94f-4b83-b450-6cadbe20bf2e"}]: dispatch 2024-09-13T00:48:38.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[83127]: audit 2024-09-13T00:48:37.571865+0000 mon.a (mon.0) 1187 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/4d2838e7-a94f-4b83-b450-6cadbe20bf2e"}]': finished 2024-09-13T00:48:38.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[83127]: audit 2024-09-13T00:48:37.591823+0000 mon.a (mon.0) 1188 : audit [DBG] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/18/6f621f21-009f-477f-9526-7f8242e7000c"}]: dispatch 2024-09-13T00:48:38.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[83127]: audit 2024-09-13T00:48:37.600691+0000 mon.a (mon.0) 1189 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' 2024-09-13T00:48:38.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[83127]: audit 2024-09-13T00:48:37.601467+0000 mon.a (mon.0) 1190 : audit [DBG] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/18/6f621f21-009f-477f-9526-7f8242e7000c"}]: dispatch 2024-09-13T00:48:38.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[83127]: audit 2024-09-13T00:48:37.602256+0000 mon.a (mon.0) 1191 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/6f621f21-009f-477f-9526-7f8242e7000c"}]: dispatch 2024-09-13T00:48:38.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[83127]: audit 2024-09-13T00:48:37.609916+0000 mon.a (mon.0) 1192 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/6f621f21-009f-477f-9526-7f8242e7000c"}]': finished 2024-09-13T00:48:38.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:38 smithi088 bash[68960]: cluster 2024-09-13T00:48:36.672768+0000 mgr.y (mgr.25304) 382 : cluster [DBG] pgmap v743: 617 pgs: 617 active+clean; 141 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 695 KiB/s rd, 593 KiB/s wr, 965 op/s 2024-09-13T00:48:38.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:38 smithi088 bash[68960]: cluster 2024-09-13T00:48:37.194152+0000 mon.a (mon.0) 1179 : cluster [DBG] osdmap e742: 8 total, 8 up, 8 in 2024-09-13T00:48:38.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:38 smithi088 bash[68960]: audit 2024-09-13T00:48:37.500794+0000 mon.a (mon.0) 1180 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]: dispatch 2024-09-13T00:48:38.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:38 smithi088 bash[68960]: audit 2024-09-13T00:48:37.509805+0000 mon.a (mon.0) 1181 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/uuid4"}]': finished 2024-09-13T00:48:38.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:38 smithi088 bash[68960]: audit 2024-09-13T00:48:37.523331+0000 mon.a (mon.0) 1182 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/e9db9f7d-1382-4c7b-a78e-58638a4a2d88"}]: dispatch 2024-09-13T00:48:38.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:38 smithi088 bash[68960]: audit 2024-09-13T00:48:37.533928+0000 mon.a (mon.0) 1183 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/e9db9f7d-1382-4c7b-a78e-58638a4a2d88"}]': finished 2024-09-13T00:48:38.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:38 smithi088 bash[68960]: audit 2024-09-13T00:48:37.552736+0000 mon.a (mon.0) 1184 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b594d1c7-285b-408a-b469-a8acf1e8fdb7"}]: dispatch 2024-09-13T00:48:38.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:38 smithi088 bash[68960]: audit 2024-09-13T00:48:37.558976+0000 mon.a (mon.0) 1185 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/b594d1c7-285b-408a-b469-a8acf1e8fdb7"}]': finished 2024-09-13T00:48:38.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:38 smithi088 bash[68960]: audit 2024-09-13T00:48:37.563813+0000 mon.a (mon.0) 1186 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/4d2838e7-a94f-4b83-b450-6cadbe20bf2e"}]: dispatch 2024-09-13T00:48:38.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:38 smithi088 bash[68960]: audit 2024-09-13T00:48:37.571865+0000 mon.a (mon.0) 1187 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/4d2838e7-a94f-4b83-b450-6cadbe20bf2e"}]': finished 2024-09-13T00:48:38.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:38 smithi088 bash[68960]: audit 2024-09-13T00:48:37.591823+0000 mon.a (mon.0) 1188 : audit [DBG] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/18/6f621f21-009f-477f-9526-7f8242e7000c"}]: dispatch 2024-09-13T00:48:38.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:38 smithi088 bash[68960]: audit 2024-09-13T00:48:37.600691+0000 mon.a (mon.0) 1189 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' 2024-09-13T00:48:38.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:38 smithi088 bash[68960]: audit 2024-09-13T00:48:37.601467+0000 mon.a (mon.0) 1190 : audit [DBG] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/18/6f621f21-009f-477f-9526-7f8242e7000c"}]: dispatch 2024-09-13T00:48:38.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:38 smithi088 bash[68960]: audit 2024-09-13T00:48:37.602256+0000 mon.a (mon.0) 1191 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/6f621f21-009f-477f-9526-7f8242e7000c"}]: dispatch 2024-09-13T00:48:38.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:38 smithi088 bash[68960]: audit 2024-09-13T00:48:37.609916+0000 mon.a (mon.0) 1192 : audit [INF] from='client.? 172.21.15.77:0/1465741529' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/18/6f621f21-009f-477f-9526-7f8242e7000c"}]': finished 2024-09-13T00:48:38.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:48:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:48:38] "GET /metrics HTTP/1.1" 200 75150 "" "Prometheus/2.43.0" 2024-09-13T00:48:40.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:39 smithi077 bash[80681]: cluster 2024-09-13T00:48:38.613987+0000 mon.a (mon.0) 1193 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2024-09-13T00:48:40.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:39 smithi077 bash[80681]: cluster 2024-09-13T00:48:38.673883+0000 mgr.y (mgr.25304) 383 : cluster [DBG] pgmap v746: 617 pgs: 617 active+clean; 144 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 1.1 MiB/s wr, 124 op/s 2024-09-13T00:48:40.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:39 smithi077 bash[83127]: cluster 2024-09-13T00:48:38.613987+0000 mon.a (mon.0) 1193 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2024-09-13T00:48:40.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:39 smithi077 bash[83127]: cluster 2024-09-13T00:48:38.673883+0000 mgr.y (mgr.25304) 383 : cluster [DBG] pgmap v746: 617 pgs: 617 active+clean; 144 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 1.1 MiB/s wr, 124 op/s 2024-09-13T00:48:40.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:39 smithi088 bash[68960]: cluster 2024-09-13T00:48:38.613987+0000 mon.a (mon.0) 1193 : cluster [DBG] osdmap e743: 8 total, 8 up, 8 in 2024-09-13T00:48:40.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:39 smithi088 bash[68960]: cluster 2024-09-13T00:48:38.673883+0000 mgr.y (mgr.25304) 383 : cluster [DBG] pgmap v746: 617 pgs: 617 active+clean; 144 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 1.1 MiB/s wr, 124 op/s 2024-09-13T00:48:41.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:40 smithi077 bash[80681]: cluster 2024-09-13T00:48:39.628365+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2024-09-13T00:48:41.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:40 smithi077 bash[83127]: cluster 2024-09-13T00:48:39.628365+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2024-09-13T00:48:41.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:40 smithi088 bash[68960]: cluster 2024-09-13T00:48:39.628365+0000 mon.a (mon.0) 1194 : cluster [DBG] osdmap e744: 8 total, 8 up, 8 in 2024-09-13T00:48:42.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:41 smithi077 bash[80681]: cluster 2024-09-13T00:48:40.633711+0000 mon.a (mon.0) 1195 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2024-09-13T00:48:42.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:41 smithi077 bash[80681]: cluster 2024-09-13T00:48:40.675062+0000 mgr.y (mgr.25304) 384 : cluster [DBG] pgmap v749: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 199 KiB/s rd, 1.6 MiB/s wr, 266 op/s 2024-09-13T00:48:42.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:41 smithi077 bash[83127]: cluster 2024-09-13T00:48:40.633711+0000 mon.a (mon.0) 1195 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2024-09-13T00:48:42.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:41 smithi077 bash[83127]: cluster 2024-09-13T00:48:40.675062+0000 mgr.y (mgr.25304) 384 : cluster [DBG] pgmap v749: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 199 KiB/s rd, 1.6 MiB/s wr, 266 op/s 2024-09-13T00:48:42.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:41 smithi088 bash[68960]: cluster 2024-09-13T00:48:40.633711+0000 mon.a (mon.0) 1195 : cluster [DBG] osdmap e745: 8 total, 8 up, 8 in 2024-09-13T00:48:42.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:41 smithi088 bash[68960]: cluster 2024-09-13T00:48:40.675062+0000 mgr.y (mgr.25304) 384 : cluster [DBG] pgmap v749: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 199 KiB/s rd, 1.6 MiB/s wr, 266 op/s 2024-09-13T00:48:42.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:48:42 smithi088 bash[61791]: ts=2024-09-13T00:48:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:48:42.757 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.CreateWithMirrorEnabled (5141 ms) 2024-09-13T00:48:42.757 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot 2024-09-13T00:48:42.757 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:43.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:42 smithi077 bash[80681]: cluster 2024-09-13T00:48:41.642435+0000 mon.a (mon.0) 1196 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2024-09-13T00:48:43.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:42 smithi077 bash[83127]: cluster 2024-09-13T00:48:41.642435+0000 mon.a (mon.0) 1196 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2024-09-13T00:48:43.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:42 smithi088 bash[68960]: cluster 2024-09-13T00:48:41.642435+0000 mon.a (mon.0) 1196 : cluster [DBG] osdmap e746: 8 total, 8 up, 8 in 2024-09-13T00:48:44.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:43 smithi077 bash[80681]: cluster 2024-09-13T00:48:42.652961+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2024-09-13T00:48:44.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:43 smithi077 bash[80681]: cluster 2024-09-13T00:48:42.676540+0000 mgr.y (mgr.25304) 385 : cluster [DBG] pgmap v752: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 160 KiB/s rd, 979 KiB/s wr, 218 op/s 2024-09-13T00:48:44.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:43 smithi077 bash[80681]: audit 2024-09-13T00:48:43.422445+0000 mon.a (mon.0) 1198 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:44.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:43 smithi077 bash[83127]: cluster 2024-09-13T00:48:42.652961+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2024-09-13T00:48:44.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:43 smithi077 bash[83127]: cluster 2024-09-13T00:48:42.676540+0000 mgr.y (mgr.25304) 385 : cluster [DBG] pgmap v752: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 160 KiB/s rd, 979 KiB/s wr, 218 op/s 2024-09-13T00:48:44.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:43 smithi077 bash[83127]: audit 2024-09-13T00:48:43.422445+0000 mon.a (mon.0) 1198 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:44.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:43 smithi088 bash[68960]: cluster 2024-09-13T00:48:42.652961+0000 mon.a (mon.0) 1197 : cluster [DBG] osdmap e747: 8 total, 8 up, 8 in 2024-09-13T00:48:44.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:43 smithi088 bash[68960]: cluster 2024-09-13T00:48:42.676540+0000 mgr.y (mgr.25304) 385 : cluster [DBG] pgmap v752: 617 pgs: 617 active+clean; 147 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 160 KiB/s rd, 979 KiB/s wr, 218 op/s 2024-09-13T00:48:44.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:43 smithi088 bash[68960]: audit 2024-09-13T00:48:43.422445+0000 mon.a (mon.0) 1198 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:44.731 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot (1974 ms) 2024-09-13T00:48:44.731 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ExclusiveLock 2024-09-13T00:48:44.731 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:44.850 INFO:tasks.workunit.client.0.smithi077.stdout:1: acquiring exclusive lock 2024-09-13T00:48:44.850 INFO:tasks.workunit.client.0.smithi077.stdout:2: acquiring exclusive lock 2024-09-13T00:48:44.856 INFO:tasks.workunit.client.0.smithi077.stdout:2: exclusive lock acquired 2024-09-13T00:48:44.900 INFO:tasks.workunit.client.0.smithi077.stdout:2: releasing exclusive lock 2024-09-13T00:48:44.905 INFO:tasks.workunit.client.0.smithi077.stdout:2: exclusive lock released 2024-09-13T00:48:44.905 INFO:tasks.workunit.client.0.smithi077.stdout:2: acquiring exclusive lock 2024-09-13T00:48:44.911 INFO:tasks.workunit.client.0.smithi077.stdout:1: exclusive lock acquired 2024-09-13T00:48:44.925 INFO:tasks.workunit.client.0.smithi077.stdout:1: releasing exclusive lock 2024-09-13T00:48:44.930 INFO:tasks.workunit.client.0.smithi077.stdout:1: exclusive lock released 2024-09-13T00:48:44.930 INFO:tasks.workunit.client.0.smithi077.stdout:1: acquiring exclusive lock 2024-09-13T00:48:44.935 INFO:tasks.workunit.client.0.smithi077.stdout:2: exclusive lock acquired 2024-09-13T00:48:44.964 INFO:tasks.workunit.client.0.smithi077.stdout:2: releasing exclusive lock 2024-09-13T00:48:44.970 INFO:tasks.workunit.client.0.smithi077.stdout:2: exclusive lock released 2024-09-13T00:48:44.970 INFO:tasks.workunit.client.0.smithi077.stdout:2: acquiring exclusive lock 2024-09-13T00:48:44.977 INFO:tasks.workunit.client.0.smithi077.stdout:1: exclusive lock acquired 2024-09-13T00:48:45.001 INFO:tasks.workunit.client.0.smithi077.stdout:1: releasing exclusive lock 2024-09-13T00:48:45.008 INFO:tasks.workunit.client.0.smithi077.stdout:1: exclusive lock released 2024-09-13T00:48:45.008 INFO:tasks.workunit.client.0.smithi077.stdout:1: acquiring exclusive lock 2024-09-13T00:48:45.013 INFO:tasks.workunit.client.0.smithi077.stdout:2: exclusive lock acquired 2024-09-13T00:48:45.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:44 smithi077 bash[80681]: cluster 2024-09-13T00:48:43.665546+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2024-09-13T00:48:45.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:44 smithi077 bash[83127]: cluster 2024-09-13T00:48:43.665546+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2024-09-13T00:48:45.029 INFO:tasks.workunit.client.0.smithi077.stdout:2: releasing exclusive lock 2024-09-13T00:48:45.035 INFO:tasks.workunit.client.0.smithi077.stdout:2: exclusive lock released 2024-09-13T00:48:45.035 INFO:tasks.workunit.client.0.smithi077.stdout:2: acquiring exclusive lock 2024-09-13T00:48:45.043 INFO:tasks.workunit.client.0.smithi077.stdout:1: exclusive lock acquired 2024-09-13T00:48:45.063 INFO:tasks.workunit.client.0.smithi077.stdout:1: releasing exclusive lock 2024-09-13T00:48:45.070 INFO:tasks.workunit.client.0.smithi077.stdout:1: exclusive lock released 2024-09-13T00:48:45.070 INFO:tasks.workunit.client.0.smithi077.stdout:1: acquiring exclusive lock 2024-09-13T00:48:45.079 INFO:tasks.workunit.client.0.smithi077.stdout:2: exclusive lock acquired 2024-09-13T00:48:45.096 INFO:tasks.workunit.client.0.smithi077.stdout:2: releasing exclusive lock 2024-09-13T00:48:45.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:44 smithi088 bash[68960]: cluster 2024-09-13T00:48:43.665546+0000 mon.a (mon.0) 1199 : cluster [DBG] osdmap e748: 8 total, 8 up, 8 in 2024-09-13T00:48:45.103 INFO:tasks.workunit.client.0.smithi077.stdout:2: exclusive lock released 2024-09-13T00:48:45.103 INFO:tasks.workunit.client.0.smithi077.stdout:2: acquiring exclusive lock 2024-09-13T00:48:45.112 INFO:tasks.workunit.client.0.smithi077.stdout:1: exclusive lock acquired 2024-09-13T00:48:45.123 INFO:tasks.workunit.client.0.smithi077.stdout:1: releasing exclusive lock 2024-09-13T00:48:45.130 INFO:tasks.workunit.client.0.smithi077.stdout:1: exclusive lock released 2024-09-13T00:48:45.130 INFO:tasks.workunit.client.0.smithi077.stdout:1: acquiring exclusive lock 2024-09-13T00:48:45.139 INFO:tasks.workunit.client.0.smithi077.stdout:1: exclusive lock acquired 2024-09-13T00:48:45.145 INFO:tasks.workunit.client.0.smithi077.stdout:1: releasing exclusive lock 2024-09-13T00:48:45.151 INFO:tasks.workunit.client.0.smithi077.stdout:1: exclusive lock released 2024-09-13T00:48:45.160 INFO:tasks.workunit.client.0.smithi077.stdout:2: exclusive lock acquired 2024-09-13T00:48:45.210 INFO:tasks.workunit.client.0.smithi077.stdout:2: releasing exclusive lock 2024-09-13T00:48:45.217 INFO:tasks.workunit.client.0.smithi077.stdout:2: exclusive lock released 2024-09-13T00:48:45.257 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ExclusiveLock (526 ms) 2024-09-13T00:48:45.257 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.BreakLock 2024-09-13T00:48:45.293 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:45.763 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.BreakLock (506 ms) 2024-09-13T00:48:45.763 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.DiscardAfterWrite 2024-09-13T00:48:45.763 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:45.824 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.DiscardAfterWrite (61 ms) 2024-09-13T00:48:45.824 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.DefaultFeatures 2024-09-13T00:48:45.824 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.DefaultFeatures (0 ms) 2024-09-13T00:48:45.824 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurge 2024-09-13T00:48:45.824 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:45.981 INFO:tasks.workunit.client.0.smithi077.stdout:0.00% done 2024-09-13T00:48:45.981 INFO:tasks.workunit.client.0.smithi077.stdout:1.56% done 2024-09-13T00:48:45.981 INFO:tasks.workunit.client.0.smithi077.stdout:3.12% done 2024-09-13T00:48:45.981 INFO:tasks.workunit.client.0.smithi077.stdout:4.69% done 2024-09-13T00:48:45.981 INFO:tasks.workunit.client.0.smithi077.stdout:6.25% done 2024-09-13T00:48:45.981 INFO:tasks.workunit.client.0.smithi077.stdout:7.81% done 2024-09-13T00:48:45.981 INFO:tasks.workunit.client.0.smithi077.stdout:9.38% done 2024-09-13T00:48:45.982 INFO:tasks.workunit.client.0.smithi077.stdout:10.94% done 2024-09-13T00:48:45.982 INFO:tasks.workunit.client.0.smithi077.stdout:12.50% done 2024-09-13T00:48:45.982 INFO:tasks.workunit.client.0.smithi077.stdout:14.06% done 2024-09-13T00:48:45.982 INFO:tasks.workunit.client.0.smithi077.stdout:15.62% done 2024-09-13T00:48:45.982 INFO:tasks.workunit.client.0.smithi077.stdout:17.19% done 2024-09-13T00:48:45.982 INFO:tasks.workunit.client.0.smithi077.stdout:18.75% done 2024-09-13T00:48:45.982 INFO:tasks.workunit.client.0.smithi077.stdout:20.31% done 2024-09-13T00:48:45.982 INFO:tasks.workunit.client.0.smithi077.stdout:21.88% done 2024-09-13T00:48:45.982 INFO:tasks.workunit.client.0.smithi077.stdout:23.44% done 2024-09-13T00:48:45.982 INFO:tasks.workunit.client.0.smithi077.stdout:25.00% done 2024-09-13T00:48:45.982 INFO:tasks.workunit.client.0.smithi077.stdout:26.56% done 2024-09-13T00:48:45.982 INFO:tasks.workunit.client.0.smithi077.stdout:28.12% done 2024-09-13T00:48:45.982 INFO:tasks.workunit.client.0.smithi077.stdout:29.69% done 2024-09-13T00:48:45.983 INFO:tasks.workunit.client.0.smithi077.stdout:31.25% done 2024-09-13T00:48:45.983 INFO:tasks.workunit.client.0.smithi077.stdout:32.81% done 2024-09-13T00:48:45.983 INFO:tasks.workunit.client.0.smithi077.stdout:34.38% done 2024-09-13T00:48:45.983 INFO:tasks.workunit.client.0.smithi077.stdout:35.94% done 2024-09-13T00:48:45.983 INFO:tasks.workunit.client.0.smithi077.stdout:37.50% done 2024-09-13T00:48:45.983 INFO:tasks.workunit.client.0.smithi077.stdout:39.06% done 2024-09-13T00:48:45.983 INFO:tasks.workunit.client.0.smithi077.stdout:40.62% done 2024-09-13T00:48:45.983 INFO:tasks.workunit.client.0.smithi077.stdout:42.19% done 2024-09-13T00:48:45.983 INFO:tasks.workunit.client.0.smithi077.stdout:43.75% done 2024-09-13T00:48:45.983 INFO:tasks.workunit.client.0.smithi077.stdout:45.31% done 2024-09-13T00:48:45.983 INFO:tasks.workunit.client.0.smithi077.stdout:46.88% done 2024-09-13T00:48:45.983 INFO:tasks.workunit.client.0.smithi077.stdout:48.44% done 2024-09-13T00:48:45.984 INFO:tasks.workunit.client.0.smithi077.stdout:50.00% done 2024-09-13T00:48:45.984 INFO:tasks.workunit.client.0.smithi077.stdout:51.56% done 2024-09-13T00:48:45.984 INFO:tasks.workunit.client.0.smithi077.stdout:53.12% done 2024-09-13T00:48:45.984 INFO:tasks.workunit.client.0.smithi077.stdout:54.69% done 2024-09-13T00:48:45.984 INFO:tasks.workunit.client.0.smithi077.stdout:56.25% done 2024-09-13T00:48:45.984 INFO:tasks.workunit.client.0.smithi077.stdout:57.81% done 2024-09-13T00:48:45.984 INFO:tasks.workunit.client.0.smithi077.stdout:59.38% done 2024-09-13T00:48:45.984 INFO:tasks.workunit.client.0.smithi077.stdout:60.94% done 2024-09-13T00:48:45.984 INFO:tasks.workunit.client.0.smithi077.stdout:62.50% done 2024-09-13T00:48:45.984 INFO:tasks.workunit.client.0.smithi077.stdout:64.06% done 2024-09-13T00:48:45.984 INFO:tasks.workunit.client.0.smithi077.stdout:65.62% done 2024-09-13T00:48:45.984 INFO:tasks.workunit.client.0.smithi077.stdout:67.19% done 2024-09-13T00:48:45.985 INFO:tasks.workunit.client.0.smithi077.stdout:68.75% done 2024-09-13T00:48:45.985 INFO:tasks.workunit.client.0.smithi077.stdout:70.31% done 2024-09-13T00:48:45.985 INFO:tasks.workunit.client.0.smithi077.stdout:71.88% done 2024-09-13T00:48:45.985 INFO:tasks.workunit.client.0.smithi077.stdout:73.44% done 2024-09-13T00:48:45.985 INFO:tasks.workunit.client.0.smithi077.stdout:75.00% done 2024-09-13T00:48:45.985 INFO:tasks.workunit.client.0.smithi077.stdout:76.56% done 2024-09-13T00:48:45.985 INFO:tasks.workunit.client.0.smithi077.stdout:78.12% done 2024-09-13T00:48:45.985 INFO:tasks.workunit.client.0.smithi077.stdout:79.69% done 2024-09-13T00:48:45.985 INFO:tasks.workunit.client.0.smithi077.stdout:81.25% done 2024-09-13T00:48:45.985 INFO:tasks.workunit.client.0.smithi077.stdout:82.81% done 2024-09-13T00:48:45.985 INFO:tasks.workunit.client.0.smithi077.stdout:84.38% done 2024-09-13T00:48:45.985 INFO:tasks.workunit.client.0.smithi077.stdout:85.94% done 2024-09-13T00:48:45.985 INFO:tasks.workunit.client.0.smithi077.stdout:87.50% done 2024-09-13T00:48:45.986 INFO:tasks.workunit.client.0.smithi077.stdout:89.06% done 2024-09-13T00:48:45.986 INFO:tasks.workunit.client.0.smithi077.stdout:90.62% done 2024-09-13T00:48:45.986 INFO:tasks.workunit.client.0.smithi077.stdout:92.19% done 2024-09-13T00:48:45.986 INFO:tasks.workunit.client.0.smithi077.stdout:93.75% done 2024-09-13T00:48:45.986 INFO:tasks.workunit.client.0.smithi077.stdout:95.31% done 2024-09-13T00:48:45.986 INFO:tasks.workunit.client.0.smithi077.stdout:96.88% done 2024-09-13T00:48:45.986 INFO:tasks.workunit.client.0.smithi077.stdout:98.44% done 2024-09-13T00:48:45.986 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurge (157 ms) 2024-09-13T00:48:45.986 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay 2024-09-13T00:48:45.986 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:46.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:45 smithi077 bash[80681]: cluster 2024-09-13T00:48:44.670611+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2024-09-13T00:48:46.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:45 smithi077 bash[80681]: cluster 2024-09-13T00:48:44.678646+0000 mgr.y (mgr.25304) 386 : cluster [DBG] pgmap v755: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 160 KiB/s rd, 76 KiB/s wr, 218 op/s 2024-09-13T00:48:46.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:45 smithi077 bash[80681]: audit 2024-09-13T00:48:45.381218+0000 mon.a (mon.0) 1201 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/4170359499"}]: dispatch 2024-09-13T00:48:46.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:45 smithi077 bash[83127]: cluster 2024-09-13T00:48:44.670611+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2024-09-13T00:48:46.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:45 smithi077 bash[83127]: cluster 2024-09-13T00:48:44.678646+0000 mgr.y (mgr.25304) 386 : cluster [DBG] pgmap v755: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 160 KiB/s rd, 76 KiB/s wr, 218 op/s 2024-09-13T00:48:46.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:45 smithi077 bash[83127]: audit 2024-09-13T00:48:45.381218+0000 mon.a (mon.0) 1201 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/4170359499"}]: dispatch 2024-09-13T00:48:46.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:45 smithi088 bash[68960]: cluster 2024-09-13T00:48:44.670611+0000 mon.a (mon.0) 1200 : cluster [DBG] osdmap e749: 8 total, 8 up, 8 in 2024-09-13T00:48:46.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:45 smithi088 bash[68960]: cluster 2024-09-13T00:48:44.678646+0000 mgr.y (mgr.25304) 386 : cluster [DBG] pgmap v755: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 160 KiB/s rd, 76 KiB/s wr, 218 op/s 2024-09-13T00:48:46.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:45 smithi088 bash[68960]: audit 2024-09-13T00:48:45.381218+0000 mon.a (mon.0) 1201 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/4170359499"}]: dispatch 2024-09-13T00:48:46.143 INFO:tasks.workunit.client.0.smithi077.stdout:0.00% done 2024-09-13T00:48:46.144 INFO:tasks.workunit.client.0.smithi077.stdout:1.56% done 2024-09-13T00:48:46.144 INFO:tasks.workunit.client.0.smithi077.stdout:3.12% done 2024-09-13T00:48:46.144 INFO:tasks.workunit.client.0.smithi077.stdout:4.69% done 2024-09-13T00:48:46.144 INFO:tasks.workunit.client.0.smithi077.stdout:6.25% done 2024-09-13T00:48:46.144 INFO:tasks.workunit.client.0.smithi077.stdout:7.81% done 2024-09-13T00:48:46.144 INFO:tasks.workunit.client.0.smithi077.stdout:9.38% done 2024-09-13T00:48:46.144 INFO:tasks.workunit.client.0.smithi077.stdout:10.94% done 2024-09-13T00:48:46.144 INFO:tasks.workunit.client.0.smithi077.stdout:12.50% done 2024-09-13T00:48:46.144 INFO:tasks.workunit.client.0.smithi077.stdout:14.06% done 2024-09-13T00:48:46.144 INFO:tasks.workunit.client.0.smithi077.stdout:15.62% done 2024-09-13T00:48:46.145 INFO:tasks.workunit.client.0.smithi077.stdout:17.19% done 2024-09-13T00:48:46.145 INFO:tasks.workunit.client.0.smithi077.stdout:18.75% done 2024-09-13T00:48:46.145 INFO:tasks.workunit.client.0.smithi077.stdout:20.31% done 2024-09-13T00:48:46.145 INFO:tasks.workunit.client.0.smithi077.stdout:21.88% done 2024-09-13T00:48:46.145 INFO:tasks.workunit.client.0.smithi077.stdout:23.44% done 2024-09-13T00:48:46.145 INFO:tasks.workunit.client.0.smithi077.stdout:25.00% done 2024-09-13T00:48:46.145 INFO:tasks.workunit.client.0.smithi077.stdout:26.56% done 2024-09-13T00:48:46.145 INFO:tasks.workunit.client.0.smithi077.stdout:28.12% done 2024-09-13T00:48:46.145 INFO:tasks.workunit.client.0.smithi077.stdout:29.69% done 2024-09-13T00:48:46.145 INFO:tasks.workunit.client.0.smithi077.stdout:31.25% done 2024-09-13T00:48:46.145 INFO:tasks.workunit.client.0.smithi077.stdout:32.81% done 2024-09-13T00:48:46.145 INFO:tasks.workunit.client.0.smithi077.stdout:34.38% done 2024-09-13T00:48:46.145 INFO:tasks.workunit.client.0.smithi077.stdout:35.94% done 2024-09-13T00:48:46.146 INFO:tasks.workunit.client.0.smithi077.stdout:37.50% done 2024-09-13T00:48:46.146 INFO:tasks.workunit.client.0.smithi077.stdout:39.06% done 2024-09-13T00:48:46.146 INFO:tasks.workunit.client.0.smithi077.stdout:40.62% done 2024-09-13T00:48:46.146 INFO:tasks.workunit.client.0.smithi077.stdout:42.19% done 2024-09-13T00:48:46.146 INFO:tasks.workunit.client.0.smithi077.stdout:43.75% done 2024-09-13T00:48:46.146 INFO:tasks.workunit.client.0.smithi077.stdout:45.31% done 2024-09-13T00:48:46.146 INFO:tasks.workunit.client.0.smithi077.stdout:46.88% done 2024-09-13T00:48:46.146 INFO:tasks.workunit.client.0.smithi077.stdout:48.44% done 2024-09-13T00:48:46.146 INFO:tasks.workunit.client.0.smithi077.stdout:50.00% done 2024-09-13T00:48:46.146 INFO:tasks.workunit.client.0.smithi077.stdout:51.56% done 2024-09-13T00:48:46.146 INFO:tasks.workunit.client.0.smithi077.stdout:53.12% done 2024-09-13T00:48:46.146 INFO:tasks.workunit.client.0.smithi077.stdout:54.69% done 2024-09-13T00:48:46.146 INFO:tasks.workunit.client.0.smithi077.stdout:56.25% done 2024-09-13T00:48:46.147 INFO:tasks.workunit.client.0.smithi077.stdout:57.81% done 2024-09-13T00:48:46.147 INFO:tasks.workunit.client.0.smithi077.stdout:59.38% done 2024-09-13T00:48:46.147 INFO:tasks.workunit.client.0.smithi077.stdout:60.94% done 2024-09-13T00:48:46.147 INFO:tasks.workunit.client.0.smithi077.stdout:62.50% done 2024-09-13T00:48:46.147 INFO:tasks.workunit.client.0.smithi077.stdout:64.06% done 2024-09-13T00:48:46.147 INFO:tasks.workunit.client.0.smithi077.stdout:65.62% done 2024-09-13T00:48:46.147 INFO:tasks.workunit.client.0.smithi077.stdout:67.19% done 2024-09-13T00:48:46.147 INFO:tasks.workunit.client.0.smithi077.stdout:68.75% done 2024-09-13T00:48:46.147 INFO:tasks.workunit.client.0.smithi077.stdout:70.31% done 2024-09-13T00:48:46.147 INFO:tasks.workunit.client.0.smithi077.stdout:71.88% done 2024-09-13T00:48:46.147 INFO:tasks.workunit.client.0.smithi077.stdout:73.44% done 2024-09-13T00:48:46.147 INFO:tasks.workunit.client.0.smithi077.stdout:75.00% done 2024-09-13T00:48:46.147 INFO:tasks.workunit.client.0.smithi077.stdout:76.56% done 2024-09-13T00:48:46.147 INFO:tasks.workunit.client.0.smithi077.stdout:78.12% done 2024-09-13T00:48:46.148 INFO:tasks.workunit.client.0.smithi077.stdout:79.69% done 2024-09-13T00:48:46.148 INFO:tasks.workunit.client.0.smithi077.stdout:81.25% done 2024-09-13T00:48:46.148 INFO:tasks.workunit.client.0.smithi077.stdout:82.81% done 2024-09-13T00:48:46.148 INFO:tasks.workunit.client.0.smithi077.stdout:84.38% done 2024-09-13T00:48:46.148 INFO:tasks.workunit.client.0.smithi077.stdout:85.94% done 2024-09-13T00:48:46.148 INFO:tasks.workunit.client.0.smithi077.stdout:87.50% done 2024-09-13T00:48:46.148 INFO:tasks.workunit.client.0.smithi077.stdout:89.06% done 2024-09-13T00:48:46.148 INFO:tasks.workunit.client.0.smithi077.stdout:90.62% done 2024-09-13T00:48:46.148 INFO:tasks.workunit.client.0.smithi077.stdout:92.19% done 2024-09-13T00:48:46.148 INFO:tasks.workunit.client.0.smithi077.stdout:93.75% done 2024-09-13T00:48:46.148 INFO:tasks.workunit.client.0.smithi077.stdout:95.31% done 2024-09-13T00:48:46.148 INFO:tasks.workunit.client.0.smithi077.stdout:96.88% done 2024-09-13T00:48:46.149 INFO:tasks.workunit.client.0.smithi077.stdout:98.44% done 2024-09-13T00:48:46.149 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay (163 ms) 2024-09-13T00:48:46.149 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestTrashPurge 2024-09-13T00:48:46.149 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:46.164 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:46.478 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestTrashPurge (334 ms) 2024-09-13T00:48:46.478 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndRestore 2024-09-13T00:48:46.478 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:46.599 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestTrashMoveAndRestore (121 ms) 2024-09-13T00:48:46.599 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestListWatchers 2024-09-13T00:48:46.599 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:46.661 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestListWatchers (62 ms) 2024-09-13T00:48:46.662 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestSetSnapById 2024-09-13T00:48:46.662 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:47.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:46 smithi077 bash[80681]: audit 2024-09-13T00:48:45.682265+0000 mon.a (mon.0) 1202 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/4170359499"}]': finished 2024-09-13T00:48:47.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:46 smithi077 bash[80681]: cluster 2024-09-13T00:48:45.682397+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2024-09-13T00:48:47.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:46 smithi077 bash[83127]: audit 2024-09-13T00:48:45.682265+0000 mon.a (mon.0) 1202 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/4170359499"}]': finished 2024-09-13T00:48:47.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:46 smithi077 bash[83127]: cluster 2024-09-13T00:48:45.682397+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2024-09-13T00:48:47.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:46 smithi088 bash[68960]: audit 2024-09-13T00:48:45.682265+0000 mon.a (mon.0) 1202 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.77:0/4170359499"}]': finished 2024-09-13T00:48:47.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:46 smithi088 bash[68960]: cluster 2024-09-13T00:48:45.682397+0000 mon.a (mon.0) 1203 : cluster [DBG] osdmap e750: 8 total, 8 up, 8 in 2024-09-13T00:48:47.759 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestSetSnapById (1098 ms) 2024-09-13T00:48:47.759 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.Namespaces 2024-09-13T00:48:47.816 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.Namespaces (58 ms) 2024-09-13T00:48:47.817 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.NamespacesPP 2024-09-13T00:48:47.892 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:47.905 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:48.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:47 smithi077 bash[80681]: cluster 2024-09-13T00:48:46.680571+0000 mgr.y (mgr.25304) 387 : cluster [DBG] pgmap v757: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 211 KiB/s rd, 139 KiB/s wr, 202 op/s 2024-09-13T00:48:48.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:47 smithi077 bash[83127]: cluster 2024-09-13T00:48:46.680571+0000 mgr.y (mgr.25304) 387 : cluster [DBG] pgmap v757: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 211 KiB/s rd, 139 KiB/s wr, 202 op/s 2024-09-13T00:48:48.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:47 smithi088 bash[68960]: cluster 2024-09-13T00:48:46.680571+0000 mgr.y (mgr.25304) 387 : cluster [DBG] pgmap v757: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 211 KiB/s rd, 139 KiB/s wr, 202 op/s 2024-09-13T00:48:48.102 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.NamespacesPP (285 ms) 2024-09-13T00:48:48.102 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.Migration 2024-09-13T00:48:48.102 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:48.102 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:48.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:48:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:48:48] "GET /metrics HTTP/1.1" 200 75152 "" "Prometheus/2.43.0" 2024-09-13T00:48:48.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:48 smithi077 bash[80681]: cluster 2024-09-13T00:48:47.697412+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2024-09-13T00:48:48.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:48 smithi077 bash[83127]: cluster 2024-09-13T00:48:47.697412+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2024-09-13T00:48:49.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:48 smithi088 bash[68960]: cluster 2024-09-13T00:48:47.697412+0000 mon.a (mon.0) 1204 : cluster [DBG] osdmap e751: 8 total, 8 up, 8 in 2024-09-13T00:48:49.245 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.Migration (1143 ms) 2024-09-13T00:48:49.246 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.MigrationPP 2024-09-13T00:48:49.246 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:49.246 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:50.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:49 smithi077 bash[83127]: cluster 2024-09-13T00:48:48.683004+0000 mgr.y (mgr.25304) 388 : cluster [DBG] pgmap v759: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 192 KiB/s rd, 112 KiB/s wr, 193 op/s 2024-09-13T00:48:50.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:49 smithi077 bash[80681]: cluster 2024-09-13T00:48:48.683004+0000 mgr.y (mgr.25304) 388 : cluster [DBG] pgmap v759: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 192 KiB/s rd, 112 KiB/s wr, 193 op/s 2024-09-13T00:48:50.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:49 smithi088 bash[68960]: cluster 2024-09-13T00:48:48.683004+0000 mgr.y (mgr.25304) 388 : cluster [DBG] pgmap v759: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 192 KiB/s rd, 112 KiB/s wr, 193 op/s 2024-09-13T00:48:50.335 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.MigrationPP (1089 ms) 2024-09-13T00:48:50.335 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestGetAccessTimestamp 2024-09-13T00:48:50.335 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:50.380 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestGetAccessTimestamp (46 ms) 2024-09-13T00:48:50.381 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.TestGetModifyTimestamp 2024-09-13T00:48:50.381 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:50.427 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.TestGetModifyTimestamp (46 ms) 2024-09-13T00:48:50.427 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ZeroOverlapFlatten 2024-09-13T00:48:50.427 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:50.982 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ZeroOverlapFlatten (555 ms) 2024-09-13T00:48:50.982 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.PoolMetadata 2024-09-13T00:48:51.099 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.PoolMetadata (117 ms) 2024-09-13T00:48:51.099 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.PoolMetadataPP 2024-09-13T00:48:51.135 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.PoolMetadataPP (36 ms) 2024-09-13T00:48:51.135 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.Config 2024-09-13T00:48:51.143 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:51.213 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.Config (78 ms) 2024-09-13T00:48:51.213 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ConfigPP 2024-09-13T00:48:51.222 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:51.326 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ConfigPP (113 ms) 2024-09-13T00:48:51.326 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.PoolStatsPP 2024-09-13T00:48:51.528 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[73137]: [13/Sep/2024:00:48:51] ENGINE Bus STOPPING 2024-09-13T00:48:51.804 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[73137]: [13/Sep/2024:00:48:51] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:48:51.804 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[73137]: [13/Sep/2024:00:48:51] ENGINE Bus STOPPED 2024-09-13T00:48:51.804 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[73137]: [13/Sep/2024:00:48:51] ENGINE Bus STARTING 2024-09-13T00:48:51.805 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[73137]: [13/Sep/2024:00:48:51] ENGINE Serving on http://:::9283 2024-09-13T00:48:51.805 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[73137]: [13/Sep/2024:00:48:51] ENGINE Bus STARTED 2024-09-13T00:48:52.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:51 smithi088 bash[68960]: cluster 2024-09-13T00:48:50.761715+0000 mon.a (mon.0) 1205 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2024-09-13T00:48:52.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:51 smithi088 bash[68960]: audit 2024-09-13T00:48:51.021777+0000 mon.a (mon.0) 1206 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' 2024-09-13T00:48:52.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:51 smithi088 bash[68960]: cluster 2024-09-13T00:48:51.043123+0000 mon.a (mon.0) 1207 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2024-09-13T00:48:52.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:51 smithi088 bash[68960]: audit 2024-09-13T00:48:51.067464+0000 mon.a (mon.0) 1208 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:48:52.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:51 smithi088 bash[68960]: audit 2024-09-13T00:48:51.078949+0000 mon.a (mon.0) 1209 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:48:52.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:51 smithi088 bash[68960]: audit 2024-09-13T00:48:51.081424+0000 mon.a (mon.0) 1210 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:48:52.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:51 smithi088 bash[68960]: audit 2024-09-13T00:48:51.091485+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:48:52.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[80681]: cluster 2024-09-13T00:48:50.761715+0000 mon.a (mon.0) 1205 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2024-09-13T00:48:52.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[80681]: audit 2024-09-13T00:48:51.021777+0000 mon.a (mon.0) 1206 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' 2024-09-13T00:48:52.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[80681]: cluster 2024-09-13T00:48:51.043123+0000 mon.a (mon.0) 1207 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2024-09-13T00:48:52.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[80681]: audit 2024-09-13T00:48:51.067464+0000 mon.a (mon.0) 1208 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:48:52.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[80681]: audit 2024-09-13T00:48:51.078949+0000 mon.a (mon.0) 1209 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:48:52.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[80681]: audit 2024-09-13T00:48:51.081424+0000 mon.a (mon.0) 1210 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:48:52.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[80681]: audit 2024-09-13T00:48:51.091485+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:48:52.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[83127]: cluster 2024-09-13T00:48:50.761715+0000 mon.a (mon.0) 1205 : cluster [DBG] osdmap e752: 8 total, 8 up, 8 in 2024-09-13T00:48:52.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[83127]: audit 2024-09-13T00:48:51.021777+0000 mon.a (mon.0) 1206 : audit [INF] from='client.? 172.21.15.77:0/3997499219' entity='client.admin' 2024-09-13T00:48:52.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[83127]: cluster 2024-09-13T00:48:51.043123+0000 mon.a (mon.0) 1207 : cluster [DBG] osdmap e753: 8 total, 8 up, 8 in 2024-09-13T00:48:52.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[83127]: audit 2024-09-13T00:48:51.067464+0000 mon.a (mon.0) 1208 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:48:52.281 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[83127]: audit 2024-09-13T00:48:51.078949+0000 mon.a (mon.0) 1209 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:48:52.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[83127]: audit 2024-09-13T00:48:51.081424+0000 mon.a (mon.0) 1210 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:48:52.282 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:51 smithi077 bash[83127]: audit 2024-09-13T00:48:51.091485+0000 mon.a (mon.0) 1211 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:48:52.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:48:52 smithi088 bash[61791]: ts=2024-09-13T00:48:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:48:53.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:52 smithi088 bash[68960]: cluster 2024-09-13T00:48:50.686694+0000 mgr.y (mgr.25304) 389 : cluster [DBG] pgmap v760: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 347 KiB/s rd, 106 KiB/s wr, 422 op/s 2024-09-13T00:48:53.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:52 smithi088 bash[68960]: cluster 2024-09-13T00:48:52.100203+0000 mon.a (mon.0) 1212 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2024-09-13T00:48:53.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:52 smithi088 bash[68960]: audit 2024-09-13T00:48:52.102766+0000 mon.a (mon.0) 1213 : audit [INF] from='client.? 172.21.15.77:0/137327747' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:48:53.119 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:53.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:52 smithi077 bash[83127]: cluster 2024-09-13T00:48:50.686694+0000 mgr.y (mgr.25304) 389 : cluster [DBG] pgmap v760: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 347 KiB/s rd, 106 KiB/s wr, 422 op/s 2024-09-13T00:48:53.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:52 smithi077 bash[83127]: cluster 2024-09-13T00:48:52.100203+0000 mon.a (mon.0) 1212 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2024-09-13T00:48:53.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:52 smithi077 bash[83127]: audit 2024-09-13T00:48:52.102766+0000 mon.a (mon.0) 1213 : audit [INF] from='client.? 172.21.15.77:0/137327747' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:48:53.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:52 smithi077 bash[80681]: cluster 2024-09-13T00:48:50.686694+0000 mgr.y (mgr.25304) 389 : cluster [DBG] pgmap v760: 617 pgs: 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 347 KiB/s rd, 106 KiB/s wr, 422 op/s 2024-09-13T00:48:53.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:52 smithi077 bash[80681]: cluster 2024-09-13T00:48:52.100203+0000 mon.a (mon.0) 1212 : cluster [DBG] osdmap e754: 8 total, 8 up, 8 in 2024-09-13T00:48:53.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:52 smithi077 bash[80681]: audit 2024-09-13T00:48:52.102766+0000 mon.a (mon.0) 1213 : audit [INF] from='client.? 172.21.15.77:0/137327747' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:48:54.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:53 smithi088 bash[68960]: cluster 2024-09-13T00:48:52.687978+0000 mgr.y (mgr.25304) 390 : cluster [DBG] pgmap v764: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 440 KiB/s rd, 28 KiB/s wr, 599 op/s 2024-09-13T00:48:54.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:53 smithi088 bash[68960]: cluster 2024-09-13T00:48:53.092239+0000 mon.a (mon.0) 1214 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:48:54.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:53 smithi088 bash[68960]: audit 2024-09-13T00:48:53.109912+0000 mon.a (mon.0) 1215 : audit [INF] from='client.? 172.21.15.77:0/137327747' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:48:54.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:53 smithi088 bash[68960]: cluster 2024-09-13T00:48:53.110415+0000 mon.a (mon.0) 1216 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2024-09-13T00:48:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:53 smithi077 bash[80681]: cluster 2024-09-13T00:48:52.687978+0000 mgr.y (mgr.25304) 390 : cluster [DBG] pgmap v764: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 440 KiB/s rd, 28 KiB/s wr, 599 op/s 2024-09-13T00:48:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:53 smithi077 bash[80681]: cluster 2024-09-13T00:48:53.092239+0000 mon.a (mon.0) 1214 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:48:54.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:53 smithi077 bash[80681]: audit 2024-09-13T00:48:53.109912+0000 mon.a (mon.0) 1215 : audit [INF] from='client.? 172.21.15.77:0/137327747' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:48:54.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:53 smithi077 bash[80681]: cluster 2024-09-13T00:48:53.110415+0000 mon.a (mon.0) 1216 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2024-09-13T00:48:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:53 smithi077 bash[83127]: cluster 2024-09-13T00:48:52.687978+0000 mgr.y (mgr.25304) 390 : cluster [DBG] pgmap v764: 649 pgs: 32 unknown, 617 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 440 KiB/s rd, 28 KiB/s wr, 599 op/s 2024-09-13T00:48:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:53 smithi077 bash[83127]: cluster 2024-09-13T00:48:53.092239+0000 mon.a (mon.0) 1214 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:48:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:53 smithi077 bash[83127]: audit 2024-09-13T00:48:53.109912+0000 mon.a (mon.0) 1215 : audit [INF] from='client.? 172.21.15.77:0/137327747' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:48:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:53 smithi077 bash[83127]: cluster 2024-09-13T00:48:53.110415+0000 mon.a (mon.0) 1216 : cluster [DBG] osdmap e755: 8 total, 8 up, 8 in 2024-09-13T00:48:55.188 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:55.208 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:55.230 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:55.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:55 smithi077 bash[80681]: cluster 2024-09-13T00:48:54.115872+0000 mon.a (mon.0) 1217 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2024-09-13T00:48:55.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:55 smithi077 bash[83127]: cluster 2024-09-13T00:48:54.115872+0000 mon.a (mon.0) 1217 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2024-09-13T00:48:55.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:55 smithi088 bash[68960]: cluster 2024-09-13T00:48:54.115872+0000 mon.a (mon.0) 1217 : cluster [DBG] osdmap e756: 8 total, 8 up, 8 in 2024-09-13T00:48:56.308 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.PoolStatsPP (4982 ms) 2024-09-13T00:48:56.308 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ImageSpec 2024-09-13T00:48:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:56 smithi077 bash[80681]: cluster 2024-09-13T00:48:54.691710+0000 mgr.y (mgr.25304) 391 : cluster [DBG] pgmap v767: 649 pgs: 5 creating+peering, 2 unknown, 642 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 471 KiB/s rd, 30 KiB/s wr, 608 op/s 2024-09-13T00:48:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:56 smithi077 bash[80681]: cluster 2024-09-13T00:48:55.107182+0000 mon.a (mon.0) 1218 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:48:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:56 smithi077 bash[80681]: cluster 2024-09-13T00:48:55.128309+0000 mon.a (mon.0) 1219 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2024-09-13T00:48:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:56 smithi077 bash[83127]: cluster 2024-09-13T00:48:54.691710+0000 mgr.y (mgr.25304) 391 : cluster [DBG] pgmap v767: 649 pgs: 5 creating+peering, 2 unknown, 642 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 471 KiB/s rd, 30 KiB/s wr, 608 op/s 2024-09-13T00:48:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:56 smithi077 bash[83127]: cluster 2024-09-13T00:48:55.107182+0000 mon.a (mon.0) 1218 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:48:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:56 smithi077 bash[83127]: cluster 2024-09-13T00:48:55.128309+0000 mon.a (mon.0) 1219 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2024-09-13T00:48:56.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:56 smithi088 bash[68960]: cluster 2024-09-13T00:48:54.691710+0000 mgr.y (mgr.25304) 391 : cluster [DBG] pgmap v767: 649 pgs: 5 creating+peering, 2 unknown, 642 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 471 KiB/s rd, 30 KiB/s wr, 608 op/s 2024-09-13T00:48:56.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:56 smithi088 bash[68960]: cluster 2024-09-13T00:48:55.107182+0000 mon.a (mon.0) 1218 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:48:56.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:56 smithi088 bash[68960]: cluster 2024-09-13T00:48:55.128309+0000 mon.a (mon.0) 1219 : cluster [DBG] osdmap e757: 8 total, 8 up, 8 in 2024-09-13T00:48:57.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:57 smithi077 bash[80681]: cluster 2024-09-13T00:48:56.137885+0000 mon.a (mon.0) 1220 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2024-09-13T00:48:57.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:57 smithi077 bash[83127]: cluster 2024-09-13T00:48:56.137885+0000 mon.a (mon.0) 1220 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2024-09-13T00:48:57.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:57 smithi088 bash[68960]: cluster 2024-09-13T00:48:56.137885+0000 mon.a (mon.0) 1220 : cluster [DBG] osdmap e758: 8 total, 8 up, 8 in 2024-09-13T00:48:58.174 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:48:58.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:58 smithi077 bash[80681]: cluster 2024-09-13T00:48:56.693452+0000 mgr.y (mgr.25304) 392 : cluster [DBG] pgmap v770: 649 pgs: 5 creating+peering, 644 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 357 KiB/s rd, 23 KiB/s wr, 458 op/s 2024-09-13T00:48:58.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:58 smithi077 bash[80681]: cluster 2024-09-13T00:48:57.155110+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2024-09-13T00:48:58.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:58 smithi077 bash[80681]: audit 2024-09-13T00:48:57.159010+0000 mon.a (mon.0) 1222 : audit [INF] from='client.? 172.21.15.77:0/2554496967' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:48:58.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:58 smithi077 bash[83127]: cluster 2024-09-13T00:48:56.693452+0000 mgr.y (mgr.25304) 392 : cluster [DBG] pgmap v770: 649 pgs: 5 creating+peering, 644 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 357 KiB/s rd, 23 KiB/s wr, 458 op/s 2024-09-13T00:48:58.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:58 smithi077 bash[83127]: cluster 2024-09-13T00:48:57.155110+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2024-09-13T00:48:58.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:58 smithi077 bash[83127]: audit 2024-09-13T00:48:57.159010+0000 mon.a (mon.0) 1222 : audit [INF] from='client.? 172.21.15.77:0/2554496967' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:48:58.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:58 smithi088 bash[68960]: cluster 2024-09-13T00:48:56.693452+0000 mgr.y (mgr.25304) 392 : cluster [DBG] pgmap v770: 649 pgs: 5 creating+peering, 644 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 357 KiB/s rd, 23 KiB/s wr, 458 op/s 2024-09-13T00:48:58.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:58 smithi088 bash[68960]: cluster 2024-09-13T00:48:57.155110+0000 mon.a (mon.0) 1221 : cluster [DBG] osdmap e759: 8 total, 8 up, 8 in 2024-09-13T00:48:58.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:58 smithi088 bash[68960]: audit 2024-09-13T00:48:57.159010+0000 mon.a (mon.0) 1222 : audit [INF] from='client.? 172.21.15.77:0/2554496967' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:48:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:48:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:48:58] "GET /metrics HTTP/1.1" 200 79918 "" "Prometheus/2.43.0" 2024-09-13T00:48:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:59 smithi077 bash[83127]: audit 2024-09-13T00:48:58.155052+0000 mon.a (mon.0) 1223 : audit [INF] from='client.? 172.21.15.77:0/2554496967' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:48:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:59 smithi077 bash[83127]: cluster 2024-09-13T00:48:58.155444+0000 mon.a (mon.0) 1224 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2024-09-13T00:48:59.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:48:59 smithi077 bash[83127]: audit 2024-09-13T00:48:58.422666+0000 mon.a (mon.0) 1225 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:59 smithi077 bash[80681]: audit 2024-09-13T00:48:58.155052+0000 mon.a (mon.0) 1223 : audit [INF] from='client.? 172.21.15.77:0/2554496967' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:48:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:59 smithi077 bash[80681]: cluster 2024-09-13T00:48:58.155444+0000 mon.a (mon.0) 1224 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2024-09-13T00:48:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:48:59 smithi077 bash[80681]: audit 2024-09-13T00:48:58.422666+0000 mon.a (mon.0) 1225 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:48:59.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:59 smithi088 bash[68960]: audit 2024-09-13T00:48:58.155052+0000 mon.a (mon.0) 1223 : audit [INF] from='client.? 172.21.15.77:0/2554496967' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:48:59.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:59 smithi088 bash[68960]: cluster 2024-09-13T00:48:58.155444+0000 mon.a (mon.0) 1224 : cluster [DBG] osdmap e760: 8 total, 8 up, 8 in 2024-09-13T00:48:59.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:48:59 smithi088 bash[68960]: audit 2024-09-13T00:48:58.422666+0000 mon.a (mon.0) 1225 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:00.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:00 smithi077 bash[80681]: cluster 2024-09-13T00:48:58.695243+0000 mgr.y (mgr.25304) 393 : cluster [DBG] pgmap v773: 681 pgs: 27 unknown, 654 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-09-13T00:49:00.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:00 smithi077 bash[80681]: cluster 2024-09-13T00:48:59.182774+0000 mon.a (mon.0) 1226 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2024-09-13T00:49:00.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:00 smithi077 bash[83127]: cluster 2024-09-13T00:48:58.695243+0000 mgr.y (mgr.25304) 393 : cluster [DBG] pgmap v773: 681 pgs: 27 unknown, 654 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-09-13T00:49:00.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:00 smithi077 bash[83127]: cluster 2024-09-13T00:48:59.182774+0000 mon.a (mon.0) 1226 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2024-09-13T00:49:00.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:00 smithi088 bash[68960]: cluster 2024-09-13T00:48:58.695243+0000 mgr.y (mgr.25304) 393 : cluster [DBG] pgmap v773: 681 pgs: 27 unknown, 654 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.5 KiB/s rd, 1.2 KiB/s wr, 8 op/s 2024-09-13T00:49:00.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:00 smithi088 bash[68960]: cluster 2024-09-13T00:48:59.182774+0000 mon.a (mon.0) 1226 : cluster [DBG] osdmap e761: 8 total, 8 up, 8 in 2024-09-13T00:49:01.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:01 smithi077 bash[83127]: cluster 2024-09-13T00:49:00.183827+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2024-09-13T00:49:01.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:01 smithi077 bash[80681]: cluster 2024-09-13T00:49:00.183827+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2024-09-13T00:49:01.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:01 smithi088 bash[68960]: cluster 2024-09-13T00:49:00.183827+0000 mon.a (mon.0) 1227 : cluster [DBG] osdmap e762: 8 total, 8 up, 8 in 2024-09-13T00:49:02.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:02 smithi077 bash[80681]: cluster 2024-09-13T00:49:00.696399+0000 mgr.y (mgr.25304) 394 : cluster [DBG] pgmap v776: 681 pgs: 6 unknown, 675 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 6.2 KiB/s wr, 60 op/s 2024-09-13T00:49:02.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:02 smithi077 bash[80681]: cluster 2024-09-13T00:49:01.213331+0000 mon.a (mon.0) 1228 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2024-09-13T00:49:02.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:02 smithi077 bash[83127]: cluster 2024-09-13T00:49:00.696399+0000 mgr.y (mgr.25304) 394 : cluster [DBG] pgmap v776: 681 pgs: 6 unknown, 675 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 6.2 KiB/s wr, 60 op/s 2024-09-13T00:49:02.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:02 smithi077 bash[83127]: cluster 2024-09-13T00:49:01.213331+0000 mon.a (mon.0) 1228 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2024-09-13T00:49:02.575 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ImageSpec (6267 ms) 2024-09-13T00:49:02.575 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.SnapRemoveWithChildMissing 2024-09-13T00:49:02.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:49:02 smithi088 bash[61791]: ts=2024-09-13T00:49:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:49:02.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:02 smithi088 bash[68960]: cluster 2024-09-13T00:49:00.696399+0000 mgr.y (mgr.25304) 394 : cluster [DBG] pgmap v776: 681 pgs: 6 unknown, 675 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 6.2 KiB/s wr, 60 op/s 2024-09-13T00:49:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:02 smithi088 bash[68960]: cluster 2024-09-13T00:49:01.213331+0000 mon.a (mon.0) 1228 : cluster [DBG] osdmap e763: 8 total, 8 up, 8 in 2024-09-13T00:49:03.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:03 smithi077 bash[80681]: cluster 2024-09-13T00:49:02.223656+0000 mon.a (mon.0) 1229 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2024-09-13T00:49:03.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:03 smithi077 bash[83127]: cluster 2024-09-13T00:49:02.223656+0000 mon.a (mon.0) 1229 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2024-09-13T00:49:03.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:03 smithi088 bash[68960]: cluster 2024-09-13T00:49:02.223656+0000 mon.a (mon.0) 1229 : cluster [DBG] osdmap e764: 8 total, 8 up, 8 in 2024-09-13T00:49:04.264 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:49:04.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:04 smithi077 bash[80681]: cluster 2024-09-13T00:49:02.697464+0000 mgr.y (mgr.25304) 395 : cluster [DBG] pgmap v779: 681 pgs: 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 5.0 KiB/s wr, 52 op/s 2024-09-13T00:49:04.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:04 smithi077 bash[80681]: cluster 2024-09-13T00:49:03.241772+0000 mon.a (mon.0) 1230 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2024-09-13T00:49:04.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:04 smithi077 bash[80681]: audit 2024-09-13T00:49:03.244234+0000 mon.a (mon.0) 1231 : audit [INF] from='client.? 172.21.15.77:0/3949609746' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:49:04.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:04 smithi077 bash[80681]: audit 2024-09-13T00:49:03.408290+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:04.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:04 smithi077 bash[83127]: cluster 2024-09-13T00:49:02.697464+0000 mgr.y (mgr.25304) 395 : cluster [DBG] pgmap v779: 681 pgs: 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 5.0 KiB/s wr, 52 op/s 2024-09-13T00:49:04.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:04 smithi077 bash[83127]: cluster 2024-09-13T00:49:03.241772+0000 mon.a (mon.0) 1230 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2024-09-13T00:49:04.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:04 smithi077 bash[83127]: audit 2024-09-13T00:49:03.244234+0000 mon.a (mon.0) 1231 : audit [INF] from='client.? 172.21.15.77:0/3949609746' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:49:04.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:04 smithi077 bash[83127]: audit 2024-09-13T00:49:03.408290+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:04 smithi088 bash[68960]: cluster 2024-09-13T00:49:02.697464+0000 mgr.y (mgr.25304) 395 : cluster [DBG] pgmap v779: 681 pgs: 681 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 5.0 KiB/s wr, 52 op/s 2024-09-13T00:49:04.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:04 smithi088 bash[68960]: cluster 2024-09-13T00:49:03.241772+0000 mon.a (mon.0) 1230 : cluster [DBG] osdmap e765: 8 total, 8 up, 8 in 2024-09-13T00:49:04.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:04 smithi088 bash[68960]: audit 2024-09-13T00:49:03.244234+0000 mon.a (mon.0) 1231 : audit [INF] from='client.? 172.21.15.77:0/3949609746' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:49:04.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:04 smithi088 bash[68960]: audit 2024-09-13T00:49:03.408290+0000 mon.a (mon.0) 1232 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:05.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:05 smithi077 bash[80681]: audit 2024-09-13T00:49:04.243578+0000 mon.a (mon.0) 1233 : audit [INF] from='client.? 172.21.15.77:0/3949609746' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:49:05.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:05 smithi077 bash[80681]: cluster 2024-09-13T00:49:04.244082+0000 mon.a (mon.0) 1234 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2024-09-13T00:49:05.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:05 smithi077 bash[83127]: audit 2024-09-13T00:49:04.243578+0000 mon.a (mon.0) 1233 : audit [INF] from='client.? 172.21.15.77:0/3949609746' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:49:05.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:05 smithi077 bash[83127]: cluster 2024-09-13T00:49:04.244082+0000 mon.a (mon.0) 1234 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2024-09-13T00:49:05.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:05 smithi088 bash[68960]: audit 2024-09-13T00:49:04.243578+0000 mon.a (mon.0) 1233 : audit [INF] from='client.? 172.21.15.77:0/3949609746' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:49:05.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:05 smithi088 bash[68960]: cluster 2024-09-13T00:49:04.244082+0000 mon.a (mon.0) 1234 : cluster [DBG] osdmap e766: 8 total, 8 up, 8 in 2024-09-13T00:49:06.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:06 smithi077 bash[83127]: cluster 2024-09-13T00:49:04.700744+0000 mgr.y (mgr.25304) 396 : cluster [DBG] pgmap v782: 713 pgs: 10 creating+peering, 9 unknown, 694 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 139 KiB/s rd, 6.2 KiB/s wr, 176 op/s 2024-09-13T00:49:06.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:06 smithi077 bash[83127]: cluster 2024-09-13T00:49:05.276609+0000 mon.a (mon.0) 1235 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2024-09-13T00:49:06.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:06 smithi077 bash[83127]: cluster 2024-09-13T00:49:06.046081+0000 mon.a (mon.0) 1236 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2024-09-13T00:49:06.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:06 smithi077 bash[80681]: cluster 2024-09-13T00:49:04.700744+0000 mgr.y (mgr.25304) 396 : cluster [DBG] pgmap v782: 713 pgs: 10 creating+peering, 9 unknown, 694 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 139 KiB/s rd, 6.2 KiB/s wr, 176 op/s 2024-09-13T00:49:06.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:06 smithi077 bash[80681]: cluster 2024-09-13T00:49:05.276609+0000 mon.a (mon.0) 1235 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2024-09-13T00:49:06.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:06 smithi077 bash[80681]: cluster 2024-09-13T00:49:06.046081+0000 mon.a (mon.0) 1236 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2024-09-13T00:49:06.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:06 smithi088 bash[68960]: cluster 2024-09-13T00:49:04.700744+0000 mgr.y (mgr.25304) 396 : cluster [DBG] pgmap v782: 713 pgs: 10 creating+peering, 9 unknown, 694 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 139 KiB/s rd, 6.2 KiB/s wr, 176 op/s 2024-09-13T00:49:06.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:06 smithi088 bash[68960]: cluster 2024-09-13T00:49:05.276609+0000 mon.a (mon.0) 1235 : cluster [DBG] osdmap e767: 8 total, 8 up, 8 in 2024-09-13T00:49:06.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:06 smithi088 bash[68960]: cluster 2024-09-13T00:49:06.046081+0000 mon.a (mon.0) 1236 : cluster [DBG] osdmap e768: 8 total, 8 up, 8 in 2024-09-13T00:49:08.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:08 smithi088 bash[68960]: cluster 2024-09-13T00:49:06.702813+0000 mgr.y (mgr.25304) 397 : cluster [DBG] pgmap v785: 713 pgs: 10 creating+peering, 3 unknown, 700 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 140 KiB/s rd, 6.5 KiB/s wr, 176 op/s 2024-09-13T00:49:08.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:08 smithi088 bash[68960]: cluster 2024-09-13T00:49:07.049721+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2024-09-13T00:49:08.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:08 smithi077 bash[80681]: cluster 2024-09-13T00:49:06.702813+0000 mgr.y (mgr.25304) 397 : cluster [DBG] pgmap v785: 713 pgs: 10 creating+peering, 3 unknown, 700 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 140 KiB/s rd, 6.5 KiB/s wr, 176 op/s 2024-09-13T00:49:08.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:08 smithi077 bash[80681]: cluster 2024-09-13T00:49:07.049721+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2024-09-13T00:49:08.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:08 smithi077 bash[83127]: cluster 2024-09-13T00:49:06.702813+0000 mgr.y (mgr.25304) 397 : cluster [DBG] pgmap v785: 713 pgs: 10 creating+peering, 3 unknown, 700 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 140 KiB/s rd, 6.5 KiB/s wr, 176 op/s 2024-09-13T00:49:08.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:08 smithi077 bash[83127]: cluster 2024-09-13T00:49:07.049721+0000 mon.a (mon.0) 1237 : cluster [DBG] osdmap e769: 8 total, 8 up, 8 in 2024-09-13T00:49:08.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:49:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:49:08] "GET /metrics HTTP/1.1" 200 79918 "" "Prometheus/2.43.0" 2024-09-13T00:49:09.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:09 smithi088 bash[68960]: cluster 2024-09-13T00:49:08.055691+0000 mon.a (mon.0) 1238 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2024-09-13T00:49:09.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:09 smithi077 bash[80681]: cluster 2024-09-13T00:49:08.055691+0000 mon.a (mon.0) 1238 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2024-09-13T00:49:09.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:09 smithi077 bash[83127]: cluster 2024-09-13T00:49:08.055691+0000 mon.a (mon.0) 1238 : cluster [DBG] osdmap e770: 8 total, 8 up, 8 in 2024-09-13T00:49:10.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:10 smithi088 bash[68960]: cluster 2024-09-13T00:49:08.704733+0000 mgr.y (mgr.25304) 398 : cluster [DBG] pgmap v788: 713 pgs: 5 creating+peering, 708 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 1.7 KiB/s wr, 19 op/s 2024-09-13T00:49:10.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:10 smithi088 bash[68960]: cluster 2024-09-13T00:49:09.081728+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2024-09-13T00:49:10.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:10 smithi077 bash[80681]: cluster 2024-09-13T00:49:08.704733+0000 mgr.y (mgr.25304) 398 : cluster [DBG] pgmap v788: 713 pgs: 5 creating+peering, 708 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 1.7 KiB/s wr, 19 op/s 2024-09-13T00:49:10.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:10 smithi077 bash[80681]: cluster 2024-09-13T00:49:09.081728+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2024-09-13T00:49:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:10 smithi077 bash[83127]: cluster 2024-09-13T00:49:08.704733+0000 mgr.y (mgr.25304) 398 : cluster [DBG] pgmap v788: 713 pgs: 5 creating+peering, 708 active+clean; 148 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 1.7 KiB/s wr, 19 op/s 2024-09-13T00:49:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:10 smithi077 bash[83127]: cluster 2024-09-13T00:49:09.081728+0000 mon.a (mon.0) 1239 : cluster [DBG] osdmap e771: 8 total, 8 up, 8 in 2024-09-13T00:49:11.177 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 3 2024-09-13T00:49:11.178 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 3 2024-09-13T00:49:11.178 INFO:tasks.workunit.client.0.smithi077.stdout:child: image202 2024-09-13T00:49:11.178 INFO:tasks.workunit.client.0.smithi077.stdout:child: image204 2024-09-13T00:49:11.178 INFO:tasks.workunit.client.0.smithi077.stdout:child: image203 2024-09-13T00:49:11.178 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-14/image202/85fec48a914c 2024-09-13T00:49:11.178 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:49:11.178 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-33/image203/85fe2280d3dd 2024-09-13T00:49:11.178 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:49:11.178 INFO:tasks.workunit.client.0.smithi077.stdout:found child test-librbd-smithi077-68798-14/image204/85fecdaca822 2024-09-13T00:49:11.178 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:49:11.179 INFO:tasks.workunit.client.0.smithi077.stdout:num children is: 0 2024-09-13T00:49:11.179 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 0 2024-09-13T00:49:11.179 INFO:tasks.workunit.client.0.smithi077.stdout:num snaps is: 0 2024-09-13T00:49:11.179 INFO:tasks.workunit.client.0.smithi077.stdout:expected: 0 2024-09-13T00:49:11.179 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.SnapRemoveWithChildMissing (8602 ms) 2024-09-13T00:49:11.179 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.QuiesceWatch 2024-09-13T00:49:11.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:11 smithi088 bash[68960]: cluster 2024-09-13T00:49:10.084291+0000 mon.a (mon.0) 1240 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2024-09-13T00:49:11.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:11 smithi077 bash[80681]: cluster 2024-09-13T00:49:10.084291+0000 mon.a (mon.0) 1240 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2024-09-13T00:49:11.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:11 smithi077 bash[83127]: cluster 2024-09-13T00:49:10.084291+0000 mon.a (mon.0) 1240 : cluster [DBG] osdmap e772: 8 total, 8 up, 8 in 2024-09-13T00:49:12.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:12 smithi077 bash[80681]: cluster 2024-09-13T00:49:10.705846+0000 mgr.y (mgr.25304) 399 : cluster [DBG] pgmap v791: 681 pgs: 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:49:12.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:12 smithi077 bash[80681]: cluster 2024-09-13T00:49:11.109899+0000 mon.a (mon.0) 1241 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2024-09-13T00:49:12.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:12 smithi077 bash[83127]: cluster 2024-09-13T00:49:10.705846+0000 mgr.y (mgr.25304) 399 : cluster [DBG] pgmap v791: 681 pgs: 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:49:12.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:12 smithi077 bash[83127]: cluster 2024-09-13T00:49:11.109899+0000 mon.a (mon.0) 1241 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2024-09-13T00:49:12.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:12 smithi088 bash[68960]: cluster 2024-09-13T00:49:10.705846+0000 mgr.y (mgr.25304) 399 : cluster [DBG] pgmap v791: 681 pgs: 681 active+clean; 2.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:49:12.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:12 smithi088 bash[68960]: cluster 2024-09-13T00:49:11.109899+0000 mon.a (mon.0) 1241 : cluster [DBG] osdmap e773: 8 total, 8 up, 8 in 2024-09-13T00:49:12.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:49:12 smithi088 bash[61791]: ts=2024-09-13T00:49:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:49:13.166 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:49:13.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:13 smithi077 bash[80681]: cluster 2024-09-13T00:49:12.111659+0000 mon.a (mon.0) 1242 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2024-09-13T00:49:13.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:13 smithi077 bash[80681]: audit 2024-09-13T00:49:12.117329+0000 mon.c (mon.2) 47 : audit [INF] from='client.? 172.21.15.77:0/3053332733' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:49:13.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:13 smithi077 bash[80681]: audit 2024-09-13T00:49:12.143248+0000 mon.a (mon.0) 1243 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:49:13.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:13 smithi077 bash[80681]: audit 2024-09-13T00:49:13.113149+0000 mon.a (mon.0) 1244 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:49:13.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:13 smithi077 bash[80681]: cluster 2024-09-13T00:49:13.113581+0000 mon.a (mon.0) 1245 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2024-09-13T00:49:13.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:13 smithi077 bash[83127]: cluster 2024-09-13T00:49:12.111659+0000 mon.a (mon.0) 1242 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2024-09-13T00:49:13.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:13 smithi077 bash[83127]: audit 2024-09-13T00:49:12.117329+0000 mon.c (mon.2) 47 : audit [INF] from='client.? 172.21.15.77:0/3053332733' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:49:13.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:13 smithi077 bash[83127]: audit 2024-09-13T00:49:12.143248+0000 mon.a (mon.0) 1243 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:49:13.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:13 smithi077 bash[83127]: audit 2024-09-13T00:49:13.113149+0000 mon.a (mon.0) 1244 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:49:13.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:13 smithi077 bash[83127]: cluster 2024-09-13T00:49:13.113581+0000 mon.a (mon.0) 1245 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2024-09-13T00:49:13.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:13 smithi088 bash[68960]: cluster 2024-09-13T00:49:12.111659+0000 mon.a (mon.0) 1242 : cluster [DBG] osdmap e774: 8 total, 8 up, 8 in 2024-09-13T00:49:13.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:13 smithi088 bash[68960]: audit 2024-09-13T00:49:12.117329+0000 mon.c (mon.2) 47 : audit [INF] from='client.? 172.21.15.77:0/3053332733' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:49:13.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:13 smithi088 bash[68960]: audit 2024-09-13T00:49:12.143248+0000 mon.a (mon.0) 1243 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:49:13.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:13 smithi088 bash[68960]: audit 2024-09-13T00:49:13.113149+0000 mon.a (mon.0) 1244 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:49:13.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:13 smithi088 bash[68960]: cluster 2024-09-13T00:49:13.113581+0000 mon.a (mon.0) 1245 : cluster [DBG] osdmap e775: 8 total, 8 up, 8 in 2024-09-13T00:49:14.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:14 smithi077 bash[80681]: cluster 2024-09-13T00:49:12.707097+0000 mgr.y (mgr.25304) 400 : cluster [DBG] pgmap v794: 713 pgs: 32 unknown, 681 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:49:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:14 smithi077 bash[80681]: cluster 2024-09-13T00:49:13.117240+0000 mon.a (mon.0) 1246 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:49:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:14 smithi077 bash[80681]: audit 2024-09-13T00:49:13.429504+0000 mon.a (mon.0) 1247 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:14 smithi077 bash[80681]: audit 2024-09-13T00:49:13.432102+0000 mon.a (mon.0) 1248 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:14 smithi077 bash[80681]: audit 2024-09-13T00:49:13.525081+0000 mon.a (mon.0) 1249 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:14 smithi077 bash[80681]: cluster 2024-09-13T00:49:14.139755+0000 mon.a (mon.0) 1250 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2024-09-13T00:49:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:14 smithi077 bash[83127]: cluster 2024-09-13T00:49:12.707097+0000 mgr.y (mgr.25304) 400 : cluster [DBG] pgmap v794: 713 pgs: 32 unknown, 681 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:49:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:14 smithi077 bash[83127]: cluster 2024-09-13T00:49:13.117240+0000 mon.a (mon.0) 1246 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:49:14.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:14 smithi077 bash[83127]: audit 2024-09-13T00:49:13.429504+0000 mon.a (mon.0) 1247 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:14.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:14 smithi077 bash[83127]: audit 2024-09-13T00:49:13.432102+0000 mon.a (mon.0) 1248 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:14.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:14 smithi077 bash[83127]: audit 2024-09-13T00:49:13.525081+0000 mon.a (mon.0) 1249 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:14.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:14 smithi077 bash[83127]: cluster 2024-09-13T00:49:14.139755+0000 mon.a (mon.0) 1250 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2024-09-13T00:49:14.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:14 smithi088 bash[68960]: cluster 2024-09-13T00:49:12.707097+0000 mgr.y (mgr.25304) 400 : cluster [DBG] pgmap v794: 713 pgs: 32 unknown, 681 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:49:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:14 smithi088 bash[68960]: cluster 2024-09-13T00:49:13.117240+0000 mon.a (mon.0) 1246 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:49:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:14 smithi088 bash[68960]: audit 2024-09-13T00:49:13.429504+0000 mon.a (mon.0) 1247 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:14 smithi088 bash[68960]: audit 2024-09-13T00:49:13.432102+0000 mon.a (mon.0) 1248 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:14 smithi088 bash[68960]: audit 2024-09-13T00:49:13.525081+0000 mon.a (mon.0) 1249 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:14 smithi088 bash[68960]: cluster 2024-09-13T00:49:14.139755+0000 mon.a (mon.0) 1250 : cluster [DBG] osdmap e776: 8 total, 8 up, 8 in 2024-09-13T00:49:15.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:15 smithi077 bash[80681]: cluster 2024-09-13T00:49:15.154412+0000 mon.a (mon.0) 1251 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:49:15.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:15 smithi077 bash[80681]: cluster 2024-09-13T00:49:15.172966+0000 mon.a (mon.0) 1252 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2024-09-13T00:49:15.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:15 smithi077 bash[83127]: cluster 2024-09-13T00:49:15.154412+0000 mon.a (mon.0) 1251 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:49:15.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:15 smithi077 bash[83127]: cluster 2024-09-13T00:49:15.172966+0000 mon.a (mon.0) 1252 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2024-09-13T00:49:15.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:15 smithi088 bash[68960]: cluster 2024-09-13T00:49:15.154412+0000 mon.a (mon.0) 1251 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:49:15.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:15 smithi088 bash[68960]: cluster 2024-09-13T00:49:15.172966+0000 mon.a (mon.0) 1252 : cluster [DBG] osdmap e777: 8 total, 8 up, 8 in 2024-09-13T00:49:16.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:16 smithi077 bash[80681]: cluster 2024-09-13T00:49:14.711142+0000 mgr.y (mgr.25304) 401 : cluster [DBG] pgmap v797: 713 pgs: 6 creating+peering, 6 unknown, 701 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 124 KiB/s rd, 7.7 KiB/s wr, 158 op/s 2024-09-13T00:49:16.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:16 smithi077 bash[80681]: cluster 2024-09-13T00:49:16.051026+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2024-09-13T00:49:16.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:16 smithi077 bash[83127]: cluster 2024-09-13T00:49:14.711142+0000 mgr.y (mgr.25304) 401 : cluster [DBG] pgmap v797: 713 pgs: 6 creating+peering, 6 unknown, 701 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 124 KiB/s rd, 7.7 KiB/s wr, 158 op/s 2024-09-13T00:49:16.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:16 smithi077 bash[83127]: cluster 2024-09-13T00:49:16.051026+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2024-09-13T00:49:16.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:16 smithi088 bash[68960]: cluster 2024-09-13T00:49:14.711142+0000 mgr.y (mgr.25304) 401 : cluster [DBG] pgmap v797: 713 pgs: 6 creating+peering, 6 unknown, 701 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 124 KiB/s rd, 7.7 KiB/s wr, 158 op/s 2024-09-13T00:49:16.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:16 smithi088 bash[68960]: cluster 2024-09-13T00:49:16.051026+0000 mon.a (mon.0) 1253 : cluster [DBG] osdmap e778: 8 total, 8 up, 8 in 2024-09-13T00:49:18.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:18 smithi088 bash[68960]: cluster 2024-09-13T00:49:16.713119+0000 mgr.y (mgr.25304) 402 : cluster [DBG] pgmap v800: 713 pgs: 6 creating+peering, 707 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 6.5 KiB/s wr, 114 op/s 2024-09-13T00:49:18.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:18 smithi088 bash[68960]: cluster 2024-09-13T00:49:17.053671+0000 mon.a (mon.0) 1254 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2024-09-13T00:49:18.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:18 smithi077 bash[80681]: cluster 2024-09-13T00:49:16.713119+0000 mgr.y (mgr.25304) 402 : cluster [DBG] pgmap v800: 713 pgs: 6 creating+peering, 707 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 6.5 KiB/s wr, 114 op/s 2024-09-13T00:49:18.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:18 smithi077 bash[80681]: cluster 2024-09-13T00:49:17.053671+0000 mon.a (mon.0) 1254 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2024-09-13T00:49:18.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:18 smithi077 bash[83127]: cluster 2024-09-13T00:49:16.713119+0000 mgr.y (mgr.25304) 402 : cluster [DBG] pgmap v800: 713 pgs: 6 creating+peering, 707 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 6.5 KiB/s wr, 114 op/s 2024-09-13T00:49:18.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:18 smithi077 bash[83127]: cluster 2024-09-13T00:49:17.053671+0000 mon.a (mon.0) 1254 : cluster [DBG] osdmap e779: 8 total, 8 up, 8 in 2024-09-13T00:49:18.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:49:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:49:18] "GET /metrics HTTP/1.1" 200 82307 "" "Prometheus/2.43.0" 2024-09-13T00:49:19.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:19 smithi088 bash[68960]: cluster 2024-09-13T00:49:18.056401+0000 mon.a (mon.0) 1255 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2024-09-13T00:49:19.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:19 smithi077 bash[83127]: cluster 2024-09-13T00:49:18.056401+0000 mon.a (mon.0) 1255 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2024-09-13T00:49:19.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:19 smithi077 bash[80681]: cluster 2024-09-13T00:49:18.056401+0000 mon.a (mon.0) 1255 : cluster [DBG] osdmap e780: 8 total, 8 up, 8 in 2024-09-13T00:49:20.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:20 smithi088 bash[68960]: cluster 2024-09-13T00:49:18.715035+0000 mgr.y (mgr.25304) 403 : cluster [DBG] pgmap v803: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 1.2 KiB/s wr, 36 op/s 2024-09-13T00:49:20.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:20 smithi088 bash[68960]: cluster 2024-09-13T00:49:19.082117+0000 mon.a (mon.0) 1256 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2024-09-13T00:49:20.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:20 smithi077 bash[83127]: cluster 2024-09-13T00:49:18.715035+0000 mgr.y (mgr.25304) 403 : cluster [DBG] pgmap v803: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 1.2 KiB/s wr, 36 op/s 2024-09-13T00:49:20.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:20 smithi077 bash[83127]: cluster 2024-09-13T00:49:19.082117+0000 mon.a (mon.0) 1256 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2024-09-13T00:49:20.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:20 smithi077 bash[80681]: cluster 2024-09-13T00:49:18.715035+0000 mgr.y (mgr.25304) 403 : cluster [DBG] pgmap v803: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 1.2 KiB/s wr, 36 op/s 2024-09-13T00:49:20.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:20 smithi077 bash[80681]: cluster 2024-09-13T00:49:19.082117+0000 mon.a (mon.0) 1256 : cluster [DBG] osdmap e781: 8 total, 8 up, 8 in 2024-09-13T00:49:21.253 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.QuiesceWatch (10076 ms) 2024-09-13T00:49:21.254 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.QuiesceWatchPP 2024-09-13T00:49:21.255 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:49:21.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:21 smithi088 bash[68960]: cluster 2024-09-13T00:49:20.087880+0000 mon.a (mon.0) 1257 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2024-09-13T00:49:21.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:21 smithi088 bash[68960]: cluster 2024-09-13T00:49:21.053638+0000 mon.a (mon.0) 1258 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2024-09-13T00:49:21.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:21 smithi077 bash[83127]: cluster 2024-09-13T00:49:20.087880+0000 mon.a (mon.0) 1257 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2024-09-13T00:49:21.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:21 smithi077 bash[83127]: cluster 2024-09-13T00:49:21.053638+0000 mon.a (mon.0) 1258 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2024-09-13T00:49:21.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:21 smithi077 bash[80681]: cluster 2024-09-13T00:49:20.087880+0000 mon.a (mon.0) 1257 : cluster [DBG] osdmap e782: 8 total, 8 up, 8 in 2024-09-13T00:49:21.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:21 smithi077 bash[80681]: cluster 2024-09-13T00:49:21.053638+0000 mon.a (mon.0) 1258 : cluster [DBG] osdmap e783: 8 total, 8 up, 8 in 2024-09-13T00:49:22.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:22 smithi088 bash[68960]: cluster 2024-09-13T00:49:20.716219+0000 mgr.y (mgr.25304) 404 : cluster [DBG] pgmap v806: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 3.0 KiB/s wr, 49 op/s 2024-09-13T00:49:22.352 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:49:22 smithi088 bash[61791]: ts=2024-09-13T00:49:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:49:22.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:22 smithi077 bash[83127]: cluster 2024-09-13T00:49:20.716219+0000 mgr.y (mgr.25304) 404 : cluster [DBG] pgmap v806: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 3.0 KiB/s wr, 49 op/s 2024-09-13T00:49:22.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:22 smithi077 bash[80681]: cluster 2024-09-13T00:49:20.716219+0000 mgr.y (mgr.25304) 404 : cluster [DBG] pgmap v806: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 36 KiB/s rd, 3.0 KiB/s wr, 49 op/s 2024-09-13T00:49:23.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:23 smithi077 bash[80681]: cluster 2024-09-13T00:49:22.117011+0000 mon.a (mon.0) 1259 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2024-09-13T00:49:23.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:23 smithi077 bash[83127]: cluster 2024-09-13T00:49:22.117011+0000 mon.a (mon.0) 1259 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2024-09-13T00:49:23.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:23 smithi088 bash[68960]: cluster 2024-09-13T00:49:22.117011+0000 mon.a (mon.0) 1259 : cluster [DBG] osdmap e784: 8 total, 8 up, 8 in 2024-09-13T00:49:24.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:24 smithi077 bash[80681]: cluster 2024-09-13T00:49:22.717451+0000 mgr.y (mgr.25304) 405 : cluster [DBG] pgmap v809: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 1.7 KiB/s wr, 12 op/s 2024-09-13T00:49:24.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:24 smithi077 bash[80681]: cluster 2024-09-13T00:49:23.136072+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2024-09-13T00:49:24.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:24 smithi077 bash[80681]: audit 2024-09-13T00:49:23.581238+0000 mon.a (mon.0) 1261 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:24 smithi077 bash[83127]: cluster 2024-09-13T00:49:22.717451+0000 mgr.y (mgr.25304) 405 : cluster [DBG] pgmap v809: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 1.7 KiB/s wr, 12 op/s 2024-09-13T00:49:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:24 smithi077 bash[83127]: cluster 2024-09-13T00:49:23.136072+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2024-09-13T00:49:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:24 smithi077 bash[83127]: audit 2024-09-13T00:49:23.581238+0000 mon.a (mon.0) 1261 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:24.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:24 smithi088 bash[68960]: cluster 2024-09-13T00:49:22.717451+0000 mgr.y (mgr.25304) 405 : cluster [DBG] pgmap v809: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 1.7 KiB/s wr, 12 op/s 2024-09-13T00:49:24.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:24 smithi088 bash[68960]: cluster 2024-09-13T00:49:23.136072+0000 mon.a (mon.0) 1260 : cluster [DBG] osdmap e785: 8 total, 8 up, 8 in 2024-09-13T00:49:24.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:24 smithi088 bash[68960]: audit 2024-09-13T00:49:23.581238+0000 mon.a (mon.0) 1261 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:25.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:25 smithi077 bash[80681]: cluster 2024-09-13T00:49:24.132112+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2024-09-13T00:49:25.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:25 smithi077 bash[83127]: cluster 2024-09-13T00:49:24.132112+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2024-09-13T00:49:25.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:25 smithi088 bash[68960]: cluster 2024-09-13T00:49:24.132112+0000 mon.a (mon.0) 1262 : cluster [DBG] osdmap e786: 8 total, 8 up, 8 in 2024-09-13T00:49:26.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:26 smithi077 bash[80681]: cluster 2024-09-13T00:49:24.721571+0000 mgr.y (mgr.25304) 406 : cluster [DBG] pgmap v812: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 98 KiB/s rd, 7.7 KiB/s wr, 131 op/s 2024-09-13T00:49:26.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:26 smithi077 bash[80681]: cluster 2024-09-13T00:49:25.162767+0000 mon.a (mon.0) 1263 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2024-09-13T00:49:26.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:26 smithi077 bash[80681]: cluster 2024-09-13T00:49:26.058316+0000 mon.a (mon.0) 1264 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2024-09-13T00:49:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:26 smithi077 bash[83127]: cluster 2024-09-13T00:49:24.721571+0000 mgr.y (mgr.25304) 406 : cluster [DBG] pgmap v812: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 98 KiB/s rd, 7.7 KiB/s wr, 131 op/s 2024-09-13T00:49:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:26 smithi077 bash[83127]: cluster 2024-09-13T00:49:25.162767+0000 mon.a (mon.0) 1263 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2024-09-13T00:49:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:26 smithi077 bash[83127]: cluster 2024-09-13T00:49:26.058316+0000 mon.a (mon.0) 1264 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2024-09-13T00:49:26.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:26 smithi088 bash[68960]: cluster 2024-09-13T00:49:24.721571+0000 mgr.y (mgr.25304) 406 : cluster [DBG] pgmap v812: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 98 KiB/s rd, 7.7 KiB/s wr, 131 op/s 2024-09-13T00:49:26.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:26 smithi088 bash[68960]: cluster 2024-09-13T00:49:25.162767+0000 mon.a (mon.0) 1263 : cluster [DBG] osdmap e787: 8 total, 8 up, 8 in 2024-09-13T00:49:26.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:26 smithi088 bash[68960]: cluster 2024-09-13T00:49:26.058316+0000 mon.a (mon.0) 1264 : cluster [DBG] osdmap e788: 8 total, 8 up, 8 in 2024-09-13T00:49:27.283 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.QuiesceWatchPP (6029 ms) 2024-09-13T00:49:27.283 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.QuiesceWatchError 2024-09-13T00:49:27.283 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:49:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:28 smithi088 bash[68960]: cluster 2024-09-13T00:49:26.723556+0000 mgr.y (mgr.25304) 407 : cluster [DBG] pgmap v815: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 98 KiB/s rd, 8.0 KiB/s wr, 133 op/s 2024-09-13T00:49:28.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:28 smithi088 bash[68960]: cluster 2024-09-13T00:49:27.059650+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2024-09-13T00:49:28.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:28 smithi077 bash[83127]: cluster 2024-09-13T00:49:26.723556+0000 mgr.y (mgr.25304) 407 : cluster [DBG] pgmap v815: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 98 KiB/s rd, 8.0 KiB/s wr, 133 op/s 2024-09-13T00:49:28.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:28 smithi077 bash[83127]: cluster 2024-09-13T00:49:27.059650+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2024-09-13T00:49:28.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:28 smithi077 bash[80681]: cluster 2024-09-13T00:49:26.723556+0000 mgr.y (mgr.25304) 407 : cluster [DBG] pgmap v815: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 98 KiB/s rd, 8.0 KiB/s wr, 133 op/s 2024-09-13T00:49:28.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:28 smithi077 bash[80681]: cluster 2024-09-13T00:49:27.059650+0000 mon.a (mon.0) 1265 : cluster [DBG] osdmap e789: 8 total, 8 up, 8 in 2024-09-13T00:49:29.028 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:49:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:49:28] "GET /metrics HTTP/1.1" 200 82342 "" "Prometheus/2.43.0" 2024-09-13T00:49:29.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:29 smithi077 bash[80681]: cluster 2024-09-13T00:49:28.164314+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2024-09-13T00:49:29.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:29 smithi077 bash[80681]: audit 2024-09-13T00:49:28.429556+0000 mon.a (mon.0) 1267 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:29.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:29 smithi077 bash[80681]: audit 2024-09-13T00:49:28.431767+0000 mon.a (mon.0) 1268 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:29.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:29 smithi077 bash[83127]: cluster 2024-09-13T00:49:28.164314+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2024-09-13T00:49:29.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:29 smithi077 bash[83127]: audit 2024-09-13T00:49:28.429556+0000 mon.a (mon.0) 1267 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:29.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:29 smithi077 bash[83127]: audit 2024-09-13T00:49:28.431767+0000 mon.a (mon.0) 1268 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:29.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:29 smithi088 bash[68960]: cluster 2024-09-13T00:49:28.164314+0000 mon.a (mon.0) 1266 : cluster [DBG] osdmap e790: 8 total, 8 up, 8 in 2024-09-13T00:49:29.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:29 smithi088 bash[68960]: audit 2024-09-13T00:49:28.429556+0000 mon.a (mon.0) 1267 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:29.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:29 smithi088 bash[68960]: audit 2024-09-13T00:49:28.431767+0000 mon.a (mon.0) 1268 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:30.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:30 smithi077 bash[80681]: cluster 2024-09-13T00:49:28.725397+0000 mgr.y (mgr.25304) 408 : cluster [DBG] pgmap v818: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 1022 B/s wr, 18 op/s 2024-09-13T00:49:30.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:30 smithi077 bash[80681]: cluster 2024-09-13T00:49:29.178664+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2024-09-13T00:49:30.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:30 smithi077 bash[83127]: cluster 2024-09-13T00:49:28.725397+0000 mgr.y (mgr.25304) 408 : cluster [DBG] pgmap v818: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 1022 B/s wr, 18 op/s 2024-09-13T00:49:30.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:30 smithi077 bash[83127]: cluster 2024-09-13T00:49:29.178664+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2024-09-13T00:49:30.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:30 smithi088 bash[68960]: cluster 2024-09-13T00:49:28.725397+0000 mgr.y (mgr.25304) 408 : cluster [DBG] pgmap v818: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 1022 B/s wr, 18 op/s 2024-09-13T00:49:30.605 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:30 smithi088 bash[68960]: cluster 2024-09-13T00:49:29.178664+0000 mon.a (mon.0) 1269 : cluster [DBG] osdmap e791: 8 total, 8 up, 8 in 2024-09-13T00:49:31.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:31 smithi077 bash[80681]: cluster 2024-09-13T00:49:30.177753+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2024-09-13T00:49:31.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:31 smithi077 bash[83127]: cluster 2024-09-13T00:49:30.177753+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2024-09-13T00:49:31.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:31 smithi088 bash[68960]: cluster 2024-09-13T00:49:30.177753+0000 mon.a (mon.0) 1270 : cluster [DBG] osdmap e792: 8 total, 8 up, 8 in 2024-09-13T00:49:32.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:32 smithi077 bash[80681]: cluster 2024-09-13T00:49:30.726724+0000 mgr.y (mgr.25304) 409 : cluster [DBG] pgmap v821: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 4.2 KiB/s wr, 79 op/s 2024-09-13T00:49:32.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:32 smithi077 bash[80681]: cluster 2024-09-13T00:49:31.203472+0000 mon.a (mon.0) 1271 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2024-09-13T00:49:32.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:32 smithi077 bash[83127]: cluster 2024-09-13T00:49:30.726724+0000 mgr.y (mgr.25304) 409 : cluster [DBG] pgmap v821: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 4.2 KiB/s wr, 79 op/s 2024-09-13T00:49:32.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:32 smithi077 bash[83127]: cluster 2024-09-13T00:49:31.203472+0000 mon.a (mon.0) 1271 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2024-09-13T00:49:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:32 smithi088 bash[68960]: cluster 2024-09-13T00:49:30.726724+0000 mgr.y (mgr.25304) 409 : cluster [DBG] pgmap v821: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 4.2 KiB/s wr, 79 op/s 2024-09-13T00:49:32.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:32 smithi088 bash[68960]: cluster 2024-09-13T00:49:31.203472+0000 mon.a (mon.0) 1271 : cluster [DBG] osdmap e793: 8 total, 8 up, 8 in 2024-09-13T00:49:32.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:49:32 smithi088 bash[61791]: ts=2024-09-13T00:49:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:49:33.466 INFO:tasks.workunit.client.0.smithi077.stdout:10.00% done 2024-09-13T00:49:33.466 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.QuiesceWatchError (6183 ms) 2024-09-13T00:49:33.466 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.QuiesceWatchTimeout 2024-09-13T00:49:33.466 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:49:33.511 INFO:tasks.workunit.client.0.smithi077.stderr:test quiesce is not long enough to time out 2024-09-13T00:49:33.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:33 smithi077 bash[80681]: cluster 2024-09-13T00:49:32.211013+0000 mon.a (mon.0) 1272 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2024-09-13T00:49:33.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:33 smithi077 bash[83127]: cluster 2024-09-13T00:49:32.211013+0000 mon.a (mon.0) 1272 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2024-09-13T00:49:33.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:33 smithi088 bash[68960]: cluster 2024-09-13T00:49:32.211013+0000 mon.a (mon.0) 1272 : cluster [DBG] osdmap e794: 8 total, 8 up, 8 in 2024-09-13T00:49:34.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:34 smithi077 bash[83127]: cluster 2024-09-13T00:49:32.727893+0000 mgr.y (mgr.25304) 410 : cluster [DBG] pgmap v824: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 46 KiB/s rd, 3.7 KiB/s wr, 63 op/s 2024-09-13T00:49:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:34 smithi077 bash[83127]: cluster 2024-09-13T00:49:33.223980+0000 mon.a (mon.0) 1273 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2024-09-13T00:49:34.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:34 smithi077 bash[80681]: cluster 2024-09-13T00:49:32.727893+0000 mgr.y (mgr.25304) 410 : cluster [DBG] pgmap v824: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 46 KiB/s rd, 3.7 KiB/s wr, 63 op/s 2024-09-13T00:49:34.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:34 smithi077 bash[80681]: cluster 2024-09-13T00:49:33.223980+0000 mon.a (mon.0) 1273 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2024-09-13T00:49:34.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:34 smithi088 bash[68960]: cluster 2024-09-13T00:49:32.727893+0000 mgr.y (mgr.25304) 410 : cluster [DBG] pgmap v824: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 46 KiB/s rd, 3.7 KiB/s wr, 63 op/s 2024-09-13T00:49:34.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:34 smithi088 bash[68960]: cluster 2024-09-13T00:49:33.223980+0000 mon.a (mon.0) 1273 : cluster [DBG] osdmap e795: 8 total, 8 up, 8 in 2024-09-13T00:49:36.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:36 smithi077 bash[80681]: cluster 2024-09-13T00:49:34.732524+0000 mgr.y (mgr.25304) 411 : cluster [DBG] pgmap v826: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 5.9 KiB/s wr, 72 op/s 2024-09-13T00:49:36.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:36 smithi077 bash[80681]: cluster 2024-09-13T00:49:36.059145+0000 mon.a (mon.0) 1274 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2024-09-13T00:49:36.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:36 smithi077 bash[83127]: cluster 2024-09-13T00:49:34.732524+0000 mgr.y (mgr.25304) 411 : cluster [DBG] pgmap v826: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 5.9 KiB/s wr, 72 op/s 2024-09-13T00:49:36.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:36 smithi077 bash[83127]: cluster 2024-09-13T00:49:36.059145+0000 mon.a (mon.0) 1274 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2024-09-13T00:49:36.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:36 smithi088 bash[68960]: cluster 2024-09-13T00:49:34.732524+0000 mgr.y (mgr.25304) 411 : cluster [DBG] pgmap v826: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 5.9 KiB/s wr, 72 op/s 2024-09-13T00:49:36.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:36 smithi088 bash[68960]: cluster 2024-09-13T00:49:36.059145+0000 mon.a (mon.0) 1274 : cluster [DBG] osdmap e796: 8 total, 8 up, 8 in 2024-09-13T00:49:38.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:38 smithi077 bash[83127]: cluster 2024-09-13T00:49:36.734887+0000 mgr.y (mgr.25304) 412 : cluster [DBG] pgmap v828: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 4.9 KiB/s wr, 59 op/s 2024-09-13T00:49:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:38 smithi077 bash[80681]: cluster 2024-09-13T00:49:36.734887+0000 mgr.y (mgr.25304) 412 : cluster [DBG] pgmap v828: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 4.9 KiB/s wr, 59 op/s 2024-09-13T00:49:38.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:38 smithi088 bash[68960]: cluster 2024-09-13T00:49:36.734887+0000 mgr.y (mgr.25304) 412 : cluster [DBG] pgmap v828: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 4.9 KiB/s wr, 59 op/s 2024-09-13T00:49:38.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:49:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:49:38] "GET /metrics HTTP/1.1" 200 82342 "" "Prometheus/2.43.0" 2024-09-13T00:49:40.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:40 smithi077 bash[80681]: cluster 2024-09-13T00:49:38.737428+0000 mgr.y (mgr.25304) 413 : cluster [DBG] pgmap v829: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 4.1 KiB/s wr, 70 op/s 2024-09-13T00:49:40.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:40 smithi077 bash[83127]: cluster 2024-09-13T00:49:38.737428+0000 mgr.y (mgr.25304) 413 : cluster [DBG] pgmap v829: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 4.1 KiB/s wr, 70 op/s 2024-09-13T00:49:40.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:40 smithi088 bash[68960]: cluster 2024-09-13T00:49:38.737428+0000 mgr.y (mgr.25304) 413 : cluster [DBG] pgmap v829: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 4.1 KiB/s wr, 70 op/s 2024-09-13T00:49:42.299 INFO:tasks.workunit.client.0.smithi077.stderr:test quiesce is timed out 2024-09-13T00:49:42.304 INFO:tasks.workunit.client.0.smithi077.stderr:waiting for timed out ... 0 2024-09-13T00:49:42.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:42 smithi088 bash[68960]: cluster 2024-09-13T00:49:40.741593+0000 mgr.y (mgr.25304) 414 : cluster [DBG] pgmap v830: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 3.4 KiB/s wr, 57 op/s 2024-09-13T00:49:42.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:42 smithi088 bash[68960]: cluster 2024-09-13T00:49:41.062993+0000 mon.a (mon.0) 1275 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2024-09-13T00:49:42.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:42 smithi077 bash[80681]: cluster 2024-09-13T00:49:40.741593+0000 mgr.y (mgr.25304) 414 : cluster [DBG] pgmap v830: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 3.4 KiB/s wr, 57 op/s 2024-09-13T00:49:42.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:42 smithi077 bash[80681]: cluster 2024-09-13T00:49:41.062993+0000 mon.a (mon.0) 1275 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2024-09-13T00:49:42.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:42 smithi077 bash[83127]: cluster 2024-09-13T00:49:40.741593+0000 mgr.y (mgr.25304) 414 : cluster [DBG] pgmap v830: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 3.4 KiB/s wr, 57 op/s 2024-09-13T00:49:42.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:42 smithi077 bash[83127]: cluster 2024-09-13T00:49:41.062993+0000 mon.a (mon.0) 1275 : cluster [DBG] osdmap e797: 8 total, 8 up, 8 in 2024-09-13T00:49:42.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:49:42 smithi088 bash[61791]: ts=2024-09-13T00:49:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:49:43.305 INFO:tasks.workunit.client.0.smithi077.stderr:waiting for timed out ... 1 2024-09-13T00:49:43.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:43 smithi077 bash[83127]: cluster 2024-09-13T00:49:42.246630+0000 mon.a (mon.0) 1276 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2024-09-13T00:49:43.571 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:43 smithi077 bash[80681]: cluster 2024-09-13T00:49:42.246630+0000 mon.a (mon.0) 1276 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2024-09-13T00:49:43.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:43 smithi088 bash[68960]: cluster 2024-09-13T00:49:42.246630+0000 mon.a (mon.0) 1276 : cluster [DBG] osdmap e798: 8 total, 8 up, 8 in 2024-09-13T00:49:44.305 INFO:tasks.workunit.client.0.smithi077.stderr:waiting for timed out ... 2 2024-09-13T00:49:44.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:44 smithi077 bash[83127]: cluster 2024-09-13T00:49:42.742768+0000 mgr.y (mgr.25304) 415 : cluster [DBG] pgmap v833: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 153 B/s wr, 19 op/s 2024-09-13T00:49:44.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:44 smithi077 bash[83127]: audit 2024-09-13T00:49:43.424233+0000 mon.a (mon.0) 1277 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:44.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:44 smithi077 bash[80681]: cluster 2024-09-13T00:49:42.742768+0000 mgr.y (mgr.25304) 415 : cluster [DBG] pgmap v833: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 153 B/s wr, 19 op/s 2024-09-13T00:49:44.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:44 smithi077 bash[80681]: audit 2024-09-13T00:49:43.424233+0000 mon.a (mon.0) 1277 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:44.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:44 smithi088 bash[68960]: cluster 2024-09-13T00:49:42.742768+0000 mgr.y (mgr.25304) 415 : cluster [DBG] pgmap v833: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 153 B/s wr, 19 op/s 2024-09-13T00:49:44.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:44 smithi088 bash[68960]: audit 2024-09-13T00:49:43.424233+0000 mon.a (mon.0) 1277 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:45.305 INFO:tasks.workunit.client.0.smithi077.stderr:waiting for timed out ... 3 2024-09-13T00:49:46.305 INFO:tasks.workunit.client.0.smithi077.stderr:waiting for timed out ... 4 2024-09-13T00:49:46.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:46 smithi077 bash[80681]: cluster 2024-09-13T00:49:44.748194+0000 mgr.y (mgr.25304) 416 : cluster [DBG] pgmap v834: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 511 B/s wr, 20 op/s 2024-09-13T00:49:46.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:46 smithi077 bash[83127]: cluster 2024-09-13T00:49:44.748194+0000 mgr.y (mgr.25304) 416 : cluster [DBG] pgmap v834: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 511 B/s wr, 20 op/s 2024-09-13T00:49:46.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:46 smithi088 bash[68960]: cluster 2024-09-13T00:49:44.748194+0000 mgr.y (mgr.25304) 416 : cluster [DBG] pgmap v834: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 511 B/s wr, 20 op/s 2024-09-13T00:49:47.305 INFO:tasks.workunit.client.0.smithi077.stderr:waiting for timed out ... 5 2024-09-13T00:49:48.305 INFO:tasks.workunit.client.0.smithi077.stderr:waiting for timed out ... 6 2024-09-13T00:49:48.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:48 smithi077 bash[80681]: cluster 2024-09-13T00:49:46.750087+0000 mgr.y (mgr.25304) 417 : cluster [DBG] pgmap v835: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-09-13T00:49:48.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:48 smithi077 bash[83127]: cluster 2024-09-13T00:49:46.750087+0000 mgr.y (mgr.25304) 417 : cluster [DBG] pgmap v835: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-09-13T00:49:48.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:48 smithi088 bash[68960]: cluster 2024-09-13T00:49:46.750087+0000 mgr.y (mgr.25304) 417 : cluster [DBG] pgmap v835: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-09-13T00:49:48.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:49:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:49:48] "GET /metrics HTTP/1.1" 200 82330 "" "Prometheus/2.43.0" 2024-09-13T00:49:49.305 INFO:tasks.workunit.client.0.smithi077.stderr:waiting for timed out ... 7 2024-09-13T00:49:50.305 INFO:tasks.workunit.client.0.smithi077.stderr:waiting for timed out ... 8 2024-09-13T00:49:50.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:50 smithi077 bash[80681]: cluster 2024-09-13T00:49:48.752753+0000 mgr.y (mgr.25304) 418 : cluster [DBG] pgmap v836: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-09-13T00:49:50.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:50 smithi077 bash[83127]: cluster 2024-09-13T00:49:48.752753+0000 mgr.y (mgr.25304) 418 : cluster [DBG] pgmap v836: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-09-13T00:49:50.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:50 smithi088 bash[68960]: cluster 2024-09-13T00:49:48.752753+0000 mgr.y (mgr.25304) 418 : cluster [DBG] pgmap v836: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s 2024-09-13T00:49:51.306 INFO:tasks.workunit.client.0.smithi077.stderr:waiting for timed out ... 9 2024-09-13T00:49:51.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:51 smithi077 bash[80681]: audit 2024-09-13T00:49:51.188481+0000 mon.a (mon.0) 1278 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:49:51.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:51 smithi077 bash[83127]: audit 2024-09-13T00:49:51.188481+0000 mon.a (mon.0) 1278 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:49:51.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:51 smithi088 bash[68960]: audit 2024-09-13T00:49:51.188481+0000 mon.a (mon.0) 1278 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:49:52.306 INFO:tasks.workunit.client.0.smithi077.stderr:waiting for timed out ... 10 2024-09-13T00:49:52.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:52 smithi077 bash[83127]: cluster 2024-09-13T00:49:50.756897+0000 mgr.y (mgr.25304) 419 : cluster [DBG] pgmap v837: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 316 B/s wr, 3 op/s 2024-09-13T00:49:52.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:52 smithi077 bash[80681]: cluster 2024-09-13T00:49:50.756897+0000 mgr.y (mgr.25304) 419 : cluster [DBG] pgmap v837: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 316 B/s wr, 3 op/s 2024-09-13T00:49:52.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:52 smithi088 bash[68960]: cluster 2024-09-13T00:49:50.756897+0000 mgr.y (mgr.25304) 419 : cluster [DBG] pgmap v837: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 3.0 KiB/s rd, 316 B/s wr, 3 op/s 2024-09-13T00:49:52.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:49:52 smithi088 bash[61791]: ts=2024-09-13T00:49:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:49:54.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:54 smithi077 bash[80681]: cluster 2024-09-13T00:49:52.758854+0000 mgr.y (mgr.25304) 420 : cluster [DBG] pgmap v838: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.8 KiB/s rd, 292 B/s wr, 3 op/s 2024-09-13T00:49:54.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:54 smithi077 bash[83127]: cluster 2024-09-13T00:49:52.758854+0000 mgr.y (mgr.25304) 420 : cluster [DBG] pgmap v838: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.8 KiB/s rd, 292 B/s wr, 3 op/s 2024-09-13T00:49:54.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:54 smithi088 bash[68960]: cluster 2024-09-13T00:49:52.758854+0000 mgr.y (mgr.25304) 420 : cluster [DBG] pgmap v838: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.8 KiB/s rd, 292 B/s wr, 3 op/s 2024-09-13T00:49:56.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:56 smithi088 bash[68960]: cluster 2024-09-13T00:49:54.764192+0000 mgr.y (mgr.25304) 421 : cluster [DBG] pgmap v839: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 255 B/s wr, 3 op/s 2024-09-13T00:49:56.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:56 smithi077 bash[83127]: cluster 2024-09-13T00:49:54.764192+0000 mgr.y (mgr.25304) 421 : cluster [DBG] pgmap v839: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 255 B/s wr, 3 op/s 2024-09-13T00:49:56.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:56 smithi077 bash[80681]: cluster 2024-09-13T00:49:54.764192+0000 mgr.y (mgr.25304) 421 : cluster [DBG] pgmap v839: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.4 KiB/s rd, 255 B/s wr, 3 op/s 2024-09-13T00:49:58.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:57 smithi077 bash[80681]: cluster 2024-09-13T00:49:56.765343+0000 mgr.y (mgr.25304) 422 : cluster [DBG] pgmap v840: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:49:58.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:57 smithi077 bash[80681]: audit 2024-09-13T00:49:56.770202+0000 mon.a (mon.0) 1279 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:58.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:57 smithi077 bash[80681]: audit 2024-09-13T00:49:56.776005+0000 mon.a (mon.0) 1280 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:58.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:57 smithi077 bash[80681]: audit 2024-09-13T00:49:57.034962+0000 mon.a (mon.0) 1281 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:58.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:57 smithi077 bash[80681]: audit 2024-09-13T00:49:57.040929+0000 mon.a (mon.0) 1282 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:58.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:57 smithi077 bash[80681]: audit 2024-09-13T00:49:57.460276+0000 mon.a (mon.0) 1283 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:49:58.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:57 smithi077 bash[80681]: audit 2024-09-13T00:49:57.461955+0000 mon.a (mon.0) 1284 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:49:58.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:57 smithi077 bash[80681]: audit 2024-09-13T00:49:57.473934+0000 mon.a (mon.0) 1285 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:58.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:57 smithi077 bash[83127]: cluster 2024-09-13T00:49:56.765343+0000 mgr.y (mgr.25304) 422 : cluster [DBG] pgmap v840: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:49:58.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:57 smithi077 bash[83127]: audit 2024-09-13T00:49:56.770202+0000 mon.a (mon.0) 1279 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:58.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:57 smithi077 bash[83127]: audit 2024-09-13T00:49:56.776005+0000 mon.a (mon.0) 1280 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:58.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:57 smithi077 bash[83127]: audit 2024-09-13T00:49:57.034962+0000 mon.a (mon.0) 1281 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:58.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:57 smithi077 bash[83127]: audit 2024-09-13T00:49:57.040929+0000 mon.a (mon.0) 1282 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:58.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:57 smithi077 bash[83127]: audit 2024-09-13T00:49:57.460276+0000 mon.a (mon.0) 1283 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:49:58.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:57 smithi077 bash[83127]: audit 2024-09-13T00:49:57.461955+0000 mon.a (mon.0) 1284 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:49:58.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:57 smithi077 bash[83127]: audit 2024-09-13T00:49:57.473934+0000 mon.a (mon.0) 1285 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:58.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:57 smithi088 bash[68960]: cluster 2024-09-13T00:49:56.765343+0000 mgr.y (mgr.25304) 422 : cluster [DBG] pgmap v840: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:49:58.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:57 smithi088 bash[68960]: audit 2024-09-13T00:49:56.770202+0000 mon.a (mon.0) 1279 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:58.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:57 smithi088 bash[68960]: audit 2024-09-13T00:49:56.776005+0000 mon.a (mon.0) 1280 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:58.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:57 smithi088 bash[68960]: audit 2024-09-13T00:49:57.034962+0000 mon.a (mon.0) 1281 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:58.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:57 smithi088 bash[68960]: audit 2024-09-13T00:49:57.040929+0000 mon.a (mon.0) 1282 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:58.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:57 smithi088 bash[68960]: audit 2024-09-13T00:49:57.460276+0000 mon.a (mon.0) 1283 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:49:58.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:57 smithi088 bash[68960]: audit 2024-09-13T00:49:57.461955+0000 mon.a (mon.0) 1284 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:49:58.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:57 smithi088 bash[68960]: audit 2024-09-13T00:49:57.473934+0000 mon.a (mon.0) 1285 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:49:59.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:58 smithi077 bash[80681]: audit 2024-09-13T00:49:58.425437+0000 mon.a (mon.0) 1286 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:59.029 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:49:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:49:58] "GET /metrics HTTP/1.1" 200 82343 "" "Prometheus/2.43.0" 2024-09-13T00:49:59.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:58 smithi077 bash[83127]: audit 2024-09-13T00:49:58.425437+0000 mon.a (mon.0) 1286 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:49:59.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:58 smithi088 bash[68960]: audit 2024-09-13T00:49:58.425437+0000 mon.a (mon.0) 1286 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:00.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:49:59 smithi088 bash[68960]: cluster 2024-09-13T00:49:58.767954+0000 mgr.y (mgr.25304) 423 : cluster [DBG] pgmap v841: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:00.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:49:59 smithi077 bash[83127]: cluster 2024-09-13T00:49:58.767954+0000 mgr.y (mgr.25304) 423 : cluster [DBG] pgmap v841: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:00.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:49:59 smithi077 bash[80681]: cluster 2024-09-13T00:49:58.767954+0000 mgr.y (mgr.25304) 423 : cluster [DBG] pgmap v841: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:00.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:00 smithi077 bash[80681]: debug 2024-09-13T00:49:59.998+0000 7fc79c5c0640 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR 8 osds(s) are not reachable 2024-09-13T00:50:01.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:00 smithi088 bash[68960]: cluster 2024-09-13T00:50:00.000159+0000 mon.a (mon.0) 1287 : cluster [ERR] overall HEALTH_ERR 8 osds(s) are not reachable 2024-09-13T00:50:01.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:00 smithi077 bash[80681]: cluster 2024-09-13T00:50:00.000159+0000 mon.a (mon.0) 1287 : cluster [ERR] overall HEALTH_ERR 8 osds(s) are not reachable 2024-09-13T00:50:01.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:00 smithi077 bash[83127]: cluster 2024-09-13T00:50:00.000159+0000 mon.a (mon.0) 1287 : cluster [ERR] overall HEALTH_ERR 8 osds(s) are not reachable 2024-09-13T00:50:02.101 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:01 smithi088 bash[68960]: cluster 2024-09-13T00:50:00.772122+0000 mgr.y (mgr.25304) 424 : cluster [DBG] pgmap v842: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:02.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:01 smithi088 bash[68960]: audit 2024-09-13T00:50:01.070488+0000 mon.a (mon.0) 1288 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]: dispatch 2024-09-13T00:50:02.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:01 smithi088 bash[68960]: audit 2024-09-13T00:50:01.070995+0000 mon.a (mon.0) 1289 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.17", "id": [6, 3]}]: dispatch 2024-09-13T00:50:02.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:01 smithi088 bash[68960]: audit 2024-09-13T00:50:01.071384+0000 mon.a (mon.0) 1290 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]}]: dispatch 2024-09-13T00:50:02.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:01 smithi088 bash[68960]: audit 2024-09-13T00:50:01.071785+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.f", "id": [0, 3]}]: dispatch 2024-09-13T00:50:02.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:01 smithi088 bash[68960]: audit 2024-09-13T00:50:01.072138+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [7, 1]}]: dispatch 2024-09-13T00:50:02.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:01 smithi088 bash[68960]: audit 2024-09-13T00:50:01.072491+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.c", "id": [0, 1]}]: dispatch 2024-09-13T00:50:02.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:01 smithi077 bash[80681]: cluster 2024-09-13T00:50:00.772122+0000 mgr.y (mgr.25304) 424 : cluster [DBG] pgmap v842: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:02.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:01 smithi077 bash[80681]: audit 2024-09-13T00:50:01.070488+0000 mon.a (mon.0) 1288 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]: dispatch 2024-09-13T00:50:02.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:01 smithi077 bash[80681]: audit 2024-09-13T00:50:01.070995+0000 mon.a (mon.0) 1289 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.17", "id": [6, 3]}]: dispatch 2024-09-13T00:50:02.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:01 smithi077 bash[80681]: audit 2024-09-13T00:50:01.071384+0000 mon.a (mon.0) 1290 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]}]: dispatch 2024-09-13T00:50:02.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:01 smithi077 bash[80681]: audit 2024-09-13T00:50:01.071785+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.f", "id": [0, 3]}]: dispatch 2024-09-13T00:50:02.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:01 smithi077 bash[80681]: audit 2024-09-13T00:50:01.072138+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [7, 1]}]: dispatch 2024-09-13T00:50:02.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:01 smithi077 bash[80681]: audit 2024-09-13T00:50:01.072491+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.c", "id": [0, 1]}]: dispatch 2024-09-13T00:50:02.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:01 smithi077 bash[83127]: cluster 2024-09-13T00:50:00.772122+0000 mgr.y (mgr.25304) 424 : cluster [DBG] pgmap v842: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:02.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:01 smithi077 bash[83127]: audit 2024-09-13T00:50:01.070488+0000 mon.a (mon.0) 1288 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]: dispatch 2024-09-13T00:50:02.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:01 smithi077 bash[83127]: audit 2024-09-13T00:50:01.070995+0000 mon.a (mon.0) 1289 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.17", "id": [6, 3]}]: dispatch 2024-09-13T00:50:02.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:01 smithi077 bash[83127]: audit 2024-09-13T00:50:01.071384+0000 mon.a (mon.0) 1290 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]}]: dispatch 2024-09-13T00:50:02.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:01 smithi077 bash[83127]: audit 2024-09-13T00:50:01.071785+0000 mon.a (mon.0) 1291 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.f", "id": [0, 3]}]: dispatch 2024-09-13T00:50:02.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:01 smithi077 bash[83127]: audit 2024-09-13T00:50:01.072138+0000 mon.a (mon.0) 1292 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [7, 1]}]: dispatch 2024-09-13T00:50:02.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:01 smithi077 bash[83127]: audit 2024-09-13T00:50:01.072491+0000 mon.a (mon.0) 1293 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.c", "id": [0, 1]}]: dispatch 2024-09-13T00:50:02.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:50:02 smithi088 bash[61791]: ts=2024-09-13T00:50:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:50:03.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:02 smithi077 bash[80681]: audit 2024-09-13T00:50:01.856650+0000 mon.a (mon.0) 1294 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]': finished 2024-09-13T00:50:03.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:02 smithi077 bash[80681]: audit 2024-09-13T00:50:01.856850+0000 mon.a (mon.0) 1295 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.17", "id": [6, 3]}]': finished 2024-09-13T00:50:03.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:02 smithi077 bash[80681]: audit 2024-09-13T00:50:01.857019+0000 mon.a (mon.0) 1296 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]}]': finished 2024-09-13T00:50:03.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:02 smithi077 bash[80681]: audit 2024-09-13T00:50:01.857213+0000 mon.a (mon.0) 1297 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.f", "id": [0, 3]}]': finished 2024-09-13T00:50:03.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:02 smithi077 bash[80681]: audit 2024-09-13T00:50:01.857393+0000 mon.a (mon.0) 1298 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [7, 1]}]': finished 2024-09-13T00:50:03.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:02 smithi077 bash[80681]: audit 2024-09-13T00:50:01.857626+0000 mon.a (mon.0) 1299 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.c", "id": [0, 1]}]': finished 2024-09-13T00:50:03.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:02 smithi077 bash[80681]: cluster 2024-09-13T00:50:01.857797+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2024-09-13T00:50:03.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:02 smithi077 bash[83127]: audit 2024-09-13T00:50:01.856650+0000 mon.a (mon.0) 1294 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]': finished 2024-09-13T00:50:03.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:02 smithi077 bash[83127]: audit 2024-09-13T00:50:01.856850+0000 mon.a (mon.0) 1295 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.17", "id": [6, 3]}]': finished 2024-09-13T00:50:03.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:02 smithi077 bash[83127]: audit 2024-09-13T00:50:01.857019+0000 mon.a (mon.0) 1296 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]}]': finished 2024-09-13T00:50:03.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:02 smithi077 bash[83127]: audit 2024-09-13T00:50:01.857213+0000 mon.a (mon.0) 1297 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.f", "id": [0, 3]}]': finished 2024-09-13T00:50:03.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:02 smithi077 bash[83127]: audit 2024-09-13T00:50:01.857393+0000 mon.a (mon.0) 1298 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [7, 1]}]': finished 2024-09-13T00:50:03.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:02 smithi077 bash[83127]: audit 2024-09-13T00:50:01.857626+0000 mon.a (mon.0) 1299 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.c", "id": [0, 1]}]': finished 2024-09-13T00:50:03.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:02 smithi077 bash[83127]: cluster 2024-09-13T00:50:01.857797+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2024-09-13T00:50:03.306 INFO:tasks.workunit.client.0.smithi077.stderr:test retry succeeds 2024-09-13T00:50:03.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:02 smithi088 bash[68960]: audit 2024-09-13T00:50:01.856650+0000 mon.a (mon.0) 1294 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.1", "id": [4, 2]}]': finished 2024-09-13T00:50:03.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:02 smithi088 bash[68960]: audit 2024-09-13T00:50:01.856850+0000 mon.a (mon.0) 1295 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.17", "id": [6, 3]}]': finished 2024-09-13T00:50:03.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:02 smithi088 bash[68960]: audit 2024-09-13T00:50:01.857019+0000 mon.a (mon.0) 1296 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]}]': finished 2024-09-13T00:50:03.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:02 smithi088 bash[68960]: audit 2024-09-13T00:50:01.857213+0000 mon.a (mon.0) 1297 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.f", "id": [0, 3]}]': finished 2024-09-13T00:50:03.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:02 smithi088 bash[68960]: audit 2024-09-13T00:50:01.857393+0000 mon.a (mon.0) 1298 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.7", "id": [7, 1]}]': finished 2024-09-13T00:50:03.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:02 smithi088 bash[68960]: audit 2024-09-13T00:50:01.857626+0000 mon.a (mon.0) 1299 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.c", "id": [0, 1]}]': finished 2024-09-13T00:50:03.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:02 smithi088 bash[68960]: cluster 2024-09-13T00:50:01.857797+0000 mon.a (mon.0) 1300 : cluster [DBG] osdmap e799: 8 total, 8 up, 8 in 2024-09-13T00:50:04.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:03 smithi077 bash[80681]: cluster 2024-09-13T00:50:02.773261+0000 mgr.y (mgr.25304) 425 : cluster [DBG] pgmap v844: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:04.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:03 smithi077 bash[80681]: cluster 2024-09-13T00:50:02.866323+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2024-09-13T00:50:04.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:03 smithi077 bash[83127]: cluster 2024-09-13T00:50:02.773261+0000 mgr.y (mgr.25304) 425 : cluster [DBG] pgmap v844: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:04.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:03 smithi077 bash[83127]: cluster 2024-09-13T00:50:02.866323+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2024-09-13T00:50:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:03 smithi088 bash[68960]: cluster 2024-09-13T00:50:02.773261+0000 mgr.y (mgr.25304) 425 : cluster [DBG] pgmap v844: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:04.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:03 smithi088 bash[68960]: cluster 2024-09-13T00:50:02.866323+0000 mon.a (mon.0) 1301 : cluster [DBG] osdmap e800: 8 total, 8 up, 8 in 2024-09-13T00:50:05.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:04 smithi077 bash[80681]: cluster 2024-09-13T00:50:03.867344+0000 mon.a (mon.0) 1302 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2024-09-13T00:50:05.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:04 smithi077 bash[83127]: cluster 2024-09-13T00:50:03.867344+0000 mon.a (mon.0) 1302 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2024-09-13T00:50:05.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:04 smithi088 bash[68960]: cluster 2024-09-13T00:50:03.867344+0000 mon.a (mon.0) 1302 : cluster [DBG] osdmap e801: 8 total, 8 up, 8 in 2024-09-13T00:50:06.144 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.QuiesceWatchTimeout (32678 ms) 2024-09-13T00:50:06.145 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.WriteZeroes 2024-09-13T00:50:06.145 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:50:06.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:05 smithi077 bash[83127]: cluster 2024-09-13T00:50:04.777428+0000 mgr.y (mgr.25304) 426 : cluster [DBG] pgmap v847: 713 pgs: 1 peering, 712 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 1022 B/s wr, 8 op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T00:50:06.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:05 smithi077 bash[83127]: cluster 2024-09-13T00:50:04.897074+0000 mon.a (mon.0) 1303 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2024-09-13T00:50:06.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:05 smithi077 bash[80681]: cluster 2024-09-13T00:50:04.777428+0000 mgr.y (mgr.25304) 426 : cluster [DBG] pgmap v847: 713 pgs: 1 peering, 712 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 1022 B/s wr, 8 op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T00:50:06.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:05 smithi077 bash[80681]: cluster 2024-09-13T00:50:04.897074+0000 mon.a (mon.0) 1303 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2024-09-13T00:50:06.295 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.WriteZeroes (151 ms) 2024-09-13T00:50:06.295 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.WriteZeroesThickProvision 2024-09-13T00:50:06.295 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:50:06.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:05 smithi088 bash[68960]: cluster 2024-09-13T00:50:04.777428+0000 mgr.y (mgr.25304) 426 : cluster [DBG] pgmap v847: 713 pgs: 1 peering, 712 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 1022 B/s wr, 8 op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-13T00:50:06.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:05 smithi088 bash[68960]: cluster 2024-09-13T00:50:04.897074+0000 mon.a (mon.0) 1303 : cluster [DBG] osdmap e802: 8 total, 8 up, 8 in 2024-09-13T00:50:06.417 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.WriteZeroesThickProvision (122 ms) 2024-09-13T00:50:06.417 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] TestLibRBD.ConcurentOperations 2024-09-13T00:50:06.417 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:50:07.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:06 smithi077 bash[80681]: cluster 2024-09-13T00:50:05.902440+0000 mon.a (mon.0) 1304 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2024-09-13T00:50:07.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:06 smithi077 bash[83127]: cluster 2024-09-13T00:50:05.902440+0000 mon.a (mon.0) 1304 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2024-09-13T00:50:07.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:06 smithi088 bash[68960]: cluster 2024-09-13T00:50:05.902440+0000 mon.a (mon.0) 1304 : cluster [DBG] osdmap e803: 8 total, 8 up, 8 in 2024-09-13T00:50:08.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:07 smithi077 bash[80681]: cluster 2024-09-13T00:50:06.779546+0000 mgr.y (mgr.25304) 427 : cluster [DBG] pgmap v850: 713 pgs: 1 peering, 712 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 1.5 KiB/s wr, 12 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-09-13T00:50:08.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:07 smithi077 bash[80681]: cluster 2024-09-13T00:50:06.903904+0000 mon.a (mon.0) 1305 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2024-09-13T00:50:08.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:07 smithi077 bash[83127]: cluster 2024-09-13T00:50:06.779546+0000 mgr.y (mgr.25304) 427 : cluster [DBG] pgmap v850: 713 pgs: 1 peering, 712 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 1.5 KiB/s wr, 12 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-09-13T00:50:08.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:07 smithi077 bash[83127]: cluster 2024-09-13T00:50:06.903904+0000 mon.a (mon.0) 1305 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2024-09-13T00:50:08.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:07 smithi088 bash[68960]: cluster 2024-09-13T00:50:06.779546+0000 mgr.y (mgr.25304) 427 : cluster [DBG] pgmap v850: 713 pgs: 1 peering, 712 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 1.5 KiB/s wr, 12 op/s; 0 B/s, 2 keys/s, 0 objects/s recovering 2024-09-13T00:50:08.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:07 smithi088 bash[68960]: cluster 2024-09-13T00:50:06.903904+0000 mon.a (mon.0) 1305 : cluster [DBG] osdmap e804: 8 total, 8 up, 8 in 2024-09-13T00:50:08.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:50:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:50:08] "GET /metrics HTTP/1.1" 200 82343 "" "Prometheus/2.43.0" 2024-09-13T00:50:09.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:08 smithi077 bash[83127]: cluster 2024-09-13T00:50:07.930746+0000 mon.a (mon.0) 1306 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2024-09-13T00:50:09.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:08 smithi077 bash[80681]: cluster 2024-09-13T00:50:07.930746+0000 mon.a (mon.0) 1306 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2024-09-13T00:50:09.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:08 smithi088 bash[68960]: cluster 2024-09-13T00:50:07.930746+0000 mon.a (mon.0) 1306 : cluster [DBG] osdmap e805: 8 total, 8 up, 8 in 2024-09-13T00:50:10.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:09 smithi077 bash[80681]: cluster 2024-09-13T00:50:08.781540+0000 mgr.y (mgr.25304) 428 : cluster [DBG] pgmap v853: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 2.2 KiB/s wr, 21 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:50:10.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:09 smithi077 bash[80681]: cluster 2024-09-13T00:50:08.940647+0000 mon.a (mon.0) 1307 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2024-09-13T00:50:10.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:09 smithi077 bash[83127]: cluster 2024-09-13T00:50:08.781540+0000 mgr.y (mgr.25304) 428 : cluster [DBG] pgmap v853: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 2.2 KiB/s wr, 21 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:50:10.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:09 smithi077 bash[83127]: cluster 2024-09-13T00:50:08.940647+0000 mon.a (mon.0) 1307 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2024-09-13T00:50:10.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:09 smithi088 bash[68960]: cluster 2024-09-13T00:50:08.781540+0000 mgr.y (mgr.25304) 428 : cluster [DBG] pgmap v853: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 2.2 KiB/s wr, 21 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:50:10.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:09 smithi088 bash[68960]: cluster 2024-09-13T00:50:08.940647+0000 mon.a (mon.0) 1307 : cluster [DBG] osdmap e806: 8 total, 8 up, 8 in 2024-09-13T00:50:11.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:10 smithi077 bash[80681]: cluster 2024-09-13T00:50:09.942678+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2024-09-13T00:50:11.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:10 smithi077 bash[83127]: cluster 2024-09-13T00:50:09.942678+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2024-09-13T00:50:11.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:10 smithi088 bash[68960]: cluster 2024-09-13T00:50:09.942678+0000 mon.a (mon.0) 1308 : cluster [DBG] osdmap e807: 8 total, 8 up, 8 in 2024-09-13T00:50:12.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:11 smithi077 bash[80681]: cluster 2024-09-13T00:50:10.782829+0000 mgr.y (mgr.25304) 429 : cluster [DBG] pgmap v856: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 120 KiB/s rd, 9.2 KiB/s wr, 169 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:50:12.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:11 smithi077 bash[80681]: cluster 2024-09-13T00:50:10.971717+0000 mon.a (mon.0) 1309 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2024-09-13T00:50:12.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:11 smithi077 bash[80681]: cluster 2024-09-13T00:50:11.070470+0000 mon.a (mon.0) 1310 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2024-09-13T00:50:12.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:11 smithi077 bash[83127]: cluster 2024-09-13T00:50:10.782829+0000 mgr.y (mgr.25304) 429 : cluster [DBG] pgmap v856: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 120 KiB/s rd, 9.2 KiB/s wr, 169 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:50:12.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:11 smithi077 bash[83127]: cluster 2024-09-13T00:50:10.971717+0000 mon.a (mon.0) 1309 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2024-09-13T00:50:12.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:11 smithi077 bash[83127]: cluster 2024-09-13T00:50:11.070470+0000 mon.a (mon.0) 1310 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2024-09-13T00:50:12.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:11 smithi088 bash[68960]: cluster 2024-09-13T00:50:10.782829+0000 mgr.y (mgr.25304) 429 : cluster [DBG] pgmap v856: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 120 KiB/s rd, 9.2 KiB/s wr, 169 op/s; 0 B/s, 0 objects/s recovering 2024-09-13T00:50:12.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:11 smithi088 bash[68960]: cluster 2024-09-13T00:50:10.971717+0000 mon.a (mon.0) 1309 : cluster [DBG] osdmap e808: 8 total, 8 up, 8 in 2024-09-13T00:50:12.313 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:11 smithi088 bash[68960]: cluster 2024-09-13T00:50:11.070470+0000 mon.a (mon.0) 1310 : cluster [DBG] osdmap e809: 8 total, 8 up, 8 in 2024-09-13T00:50:12.601 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:50:12 smithi088 bash[61791]: ts=2024-09-13T00:50:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:50:13.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:13 smithi088 bash[68960]: cluster 2024-09-13T00:50:12.075190+0000 mon.a (mon.0) 1311 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2024-09-13T00:50:13.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:13 smithi077 bash[80681]: cluster 2024-09-13T00:50:12.075190+0000 mon.a (mon.0) 1311 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2024-09-13T00:50:13.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:13 smithi077 bash[83127]: cluster 2024-09-13T00:50:12.075190+0000 mon.a (mon.0) 1311 : cluster [DBG] osdmap e810: 8 total, 8 up, 8 in 2024-09-13T00:50:14.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:14 smithi088 bash[68960]: cluster 2024-09-13T00:50:12.784104+0000 mgr.y (mgr.25304) 430 : cluster [DBG] pgmap v860: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 7.8 KiB/s wr, 161 op/s 2024-09-13T00:50:14.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:14 smithi088 bash[68960]: cluster 2024-09-13T00:50:13.084764+0000 mon.a (mon.0) 1312 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2024-09-13T00:50:14.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:14 smithi088 bash[68960]: audit 2024-09-13T00:50:13.424843+0000 mon.a (mon.0) 1313 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:14.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:14 smithi088 bash[68960]: audit 2024-09-13T00:50:13.683357+0000 mon.a (mon.0) 1314 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:50:14.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:14 smithi077 bash[80681]: cluster 2024-09-13T00:50:12.784104+0000 mgr.y (mgr.25304) 430 : cluster [DBG] pgmap v860: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 7.8 KiB/s wr, 161 op/s 2024-09-13T00:50:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:14 smithi077 bash[80681]: cluster 2024-09-13T00:50:13.084764+0000 mon.a (mon.0) 1312 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2024-09-13T00:50:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:14 smithi077 bash[80681]: audit 2024-09-13T00:50:13.424843+0000 mon.a (mon.0) 1313 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:14 smithi077 bash[80681]: audit 2024-09-13T00:50:13.683357+0000 mon.a (mon.0) 1314 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:50:14.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:14 smithi077 bash[83127]: cluster 2024-09-13T00:50:12.784104+0000 mgr.y (mgr.25304) 430 : cluster [DBG] pgmap v860: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 7.8 KiB/s wr, 161 op/s 2024-09-13T00:50:14.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:14 smithi077 bash[83127]: cluster 2024-09-13T00:50:13.084764+0000 mon.a (mon.0) 1312 : cluster [DBG] osdmap e811: 8 total, 8 up, 8 in 2024-09-13T00:50:14.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:14 smithi077 bash[83127]: audit 2024-09-13T00:50:13.424843+0000 mon.a (mon.0) 1313 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:14.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:14 smithi077 bash[83127]: audit 2024-09-13T00:50:13.683357+0000 mon.a (mon.0) 1314 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:50:15.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:15 smithi088 bash[68960]: cluster 2024-09-13T00:50:14.093589+0000 mon.a (mon.0) 1315 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2024-09-13T00:50:15.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:15 smithi077 bash[80681]: cluster 2024-09-13T00:50:14.093589+0000 mon.a (mon.0) 1315 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2024-09-13T00:50:15.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:15 smithi077 bash[83127]: cluster 2024-09-13T00:50:14.093589+0000 mon.a (mon.0) 1315 : cluster [DBG] osdmap e812: 8 total, 8 up, 8 in 2024-09-13T00:50:16.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:16 smithi088 bash[68960]: cluster 2024-09-13T00:50:14.788261+0000 mgr.y (mgr.25304) 431 : cluster [DBG] pgmap v863: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.5 KiB/s wr, 22 op/s 2024-09-13T00:50:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:16 smithi088 bash[68960]: cluster 2024-09-13T00:50:15.103913+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2024-09-13T00:50:16.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:16 smithi077 bash[83127]: cluster 2024-09-13T00:50:14.788261+0000 mgr.y (mgr.25304) 431 : cluster [DBG] pgmap v863: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.5 KiB/s wr, 22 op/s 2024-09-13T00:50:16.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:16 smithi077 bash[83127]: cluster 2024-09-13T00:50:15.103913+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2024-09-13T00:50:16.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:16 smithi077 bash[80681]: cluster 2024-09-13T00:50:14.788261+0000 mgr.y (mgr.25304) 431 : cluster [DBG] pgmap v863: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.5 KiB/s wr, 22 op/s 2024-09-13T00:50:16.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:16 smithi077 bash[80681]: cluster 2024-09-13T00:50:15.103913+0000 mon.a (mon.0) 1316 : cluster [DBG] osdmap e813: 8 total, 8 up, 8 in 2024-09-13T00:50:17.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:17 smithi077 bash[80681]: cluster 2024-09-13T00:50:16.111845+0000 mon.a (mon.0) 1317 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2024-09-13T00:50:17.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:17 smithi077 bash[83127]: cluster 2024-09-13T00:50:16.111845+0000 mon.a (mon.0) 1317 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2024-09-13T00:50:17.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:17 smithi088 bash[68960]: cluster 2024-09-13T00:50:16.111845+0000 mon.a (mon.0) 1317 : cluster [DBG] osdmap e814: 8 total, 8 up, 8 in 2024-09-13T00:50:18.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:18 smithi077 bash[83127]: cluster 2024-09-13T00:50:16.790331+0000 mgr.y (mgr.25304) 432 : cluster [DBG] pgmap v866: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 4.7 KiB/s wr, 18 op/s 2024-09-13T00:50:18.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:18 smithi077 bash[83127]: cluster 2024-09-13T00:50:17.122944+0000 mon.a (mon.0) 1318 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2024-09-13T00:50:18.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:18 smithi077 bash[80681]: cluster 2024-09-13T00:50:16.790331+0000 mgr.y (mgr.25304) 432 : cluster [DBG] pgmap v866: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 4.7 KiB/s wr, 18 op/s 2024-09-13T00:50:18.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:18 smithi077 bash[80681]: cluster 2024-09-13T00:50:17.122944+0000 mon.a (mon.0) 1318 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2024-09-13T00:50:18.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:18 smithi088 bash[68960]: cluster 2024-09-13T00:50:16.790331+0000 mgr.y (mgr.25304) 432 : cluster [DBG] pgmap v866: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 4.7 KiB/s wr, 18 op/s 2024-09-13T00:50:18.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:18 smithi088 bash[68960]: cluster 2024-09-13T00:50:17.122944+0000 mon.a (mon.0) 1318 : cluster [DBG] osdmap e815: 8 total, 8 up, 8 in 2024-09-13T00:50:18.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:50:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:50:18] "GET /metrics HTTP/1.1" 200 82351 "" "Prometheus/2.43.0" 2024-09-13T00:50:19.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:19 smithi077 bash[80681]: cluster 2024-09-13T00:50:18.124253+0000 mon.a (mon.0) 1319 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2024-09-13T00:50:19.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:19 smithi077 bash[83127]: cluster 2024-09-13T00:50:18.124253+0000 mon.a (mon.0) 1319 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2024-09-13T00:50:19.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:19 smithi088 bash[68960]: cluster 2024-09-13T00:50:18.124253+0000 mon.a (mon.0) 1319 : cluster [DBG] osdmap e816: 8 total, 8 up, 8 in 2024-09-13T00:50:20.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:20 smithi077 bash[80681]: cluster 2024-09-13T00:50:18.792187+0000 mgr.y (mgr.25304) 433 : cluster [DBG] pgmap v869: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:50:20.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:20 smithi077 bash[80681]: cluster 2024-09-13T00:50:19.146755+0000 mon.a (mon.0) 1320 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2024-09-13T00:50:20.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:20 smithi077 bash[83127]: cluster 2024-09-13T00:50:18.792187+0000 mgr.y (mgr.25304) 433 : cluster [DBG] pgmap v869: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:50:20.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:20 smithi077 bash[83127]: cluster 2024-09-13T00:50:19.146755+0000 mon.a (mon.0) 1320 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2024-09-13T00:50:20.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:20 smithi088 bash[68960]: cluster 2024-09-13T00:50:18.792187+0000 mgr.y (mgr.25304) 433 : cluster [DBG] pgmap v869: 713 pgs: 713 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 0 B/s wr, 5 op/s 2024-09-13T00:50:20.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:20 smithi088 bash[68960]: cluster 2024-09-13T00:50:19.146755+0000 mon.a (mon.0) 1320 : cluster [DBG] osdmap e817: 8 total, 8 up, 8 in 2024-09-13T00:50:21.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:21 smithi077 bash[80681]: cluster 2024-09-13T00:50:20.154986+0000 mon.a (mon.0) 1321 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2024-09-13T00:50:21.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:21 smithi077 bash[83127]: cluster 2024-09-13T00:50:20.154986+0000 mon.a (mon.0) 1321 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2024-09-13T00:50:21.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:21 smithi088 bash[68960]: cluster 2024-09-13T00:50:20.154986+0000 mon.a (mon.0) 1321 : cluster [DBG] osdmap e818: 8 total, 8 up, 8 in 2024-09-13T00:50:22.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:22 smithi077 bash[80681]: cluster 2024-09-13T00:50:20.793350+0000 mgr.y (mgr.25304) 434 : cluster [DBG] pgmap v872: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 178 KiB/s rd, 5.5 KiB/s wr, 245 op/s 2024-09-13T00:50:22.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:22 smithi077 bash[80681]: cluster 2024-09-13T00:50:21.168096+0000 mon.a (mon.0) 1322 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2024-09-13T00:50:22.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:22 smithi077 bash[83127]: cluster 2024-09-13T00:50:20.793350+0000 mgr.y (mgr.25304) 434 : cluster [DBG] pgmap v872: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 178 KiB/s rd, 5.5 KiB/s wr, 245 op/s 2024-09-13T00:50:22.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:22 smithi077 bash[83127]: cluster 2024-09-13T00:50:21.168096+0000 mon.a (mon.0) 1322 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2024-09-13T00:50:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:22 smithi088 bash[68960]: cluster 2024-09-13T00:50:20.793350+0000 mgr.y (mgr.25304) 434 : cluster [DBG] pgmap v872: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 178 KiB/s rd, 5.5 KiB/s wr, 245 op/s 2024-09-13T00:50:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:22 smithi088 bash[68960]: cluster 2024-09-13T00:50:21.168096+0000 mon.a (mon.0) 1322 : cluster [DBG] osdmap e819: 8 total, 8 up, 8 in 2024-09-13T00:50:22.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:50:22 smithi088 bash[61791]: ts=2024-09-13T00:50:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:50:23.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:23 smithi077 bash[80681]: cluster 2024-09-13T00:50:22.174580+0000 mon.a (mon.0) 1323 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2024-09-13T00:50:23.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:23 smithi077 bash[83127]: cluster 2024-09-13T00:50:22.174580+0000 mon.a (mon.0) 1323 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2024-09-13T00:50:23.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:23 smithi088 bash[68960]: cluster 2024-09-13T00:50:22.174580+0000 mon.a (mon.0) 1323 : cluster [DBG] osdmap e820: 8 total, 8 up, 8 in 2024-09-13T00:50:24.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:24 smithi077 bash[80681]: cluster 2024-09-13T00:50:22.794676+0000 mgr.y (mgr.25304) 435 : cluster [DBG] pgmap v875: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 5.5 KiB/s wr, 243 op/s 2024-09-13T00:50:24.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:24 smithi077 bash[80681]: cluster 2024-09-13T00:50:23.187478+0000 mon.a (mon.0) 1324 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2024-09-13T00:50:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:24 smithi077 bash[83127]: cluster 2024-09-13T00:50:22.794676+0000 mgr.y (mgr.25304) 435 : cluster [DBG] pgmap v875: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 5.5 KiB/s wr, 243 op/s 2024-09-13T00:50:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:24 smithi077 bash[83127]: cluster 2024-09-13T00:50:23.187478+0000 mon.a (mon.0) 1324 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2024-09-13T00:50:24.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:24 smithi088 bash[68960]: cluster 2024-09-13T00:50:22.794676+0000 mgr.y (mgr.25304) 435 : cluster [DBG] pgmap v875: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 5.5 KiB/s wr, 243 op/s 2024-09-13T00:50:24.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:24 smithi088 bash[68960]: cluster 2024-09-13T00:50:23.187478+0000 mon.a (mon.0) 1324 : cluster [DBG] osdmap e821: 8 total, 8 up, 8 in 2024-09-13T00:50:25.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:25 smithi077 bash[80681]: cluster 2024-09-13T00:50:24.189870+0000 mon.a (mon.0) 1325 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2024-09-13T00:50:25.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:25 smithi077 bash[83127]: cluster 2024-09-13T00:50:24.189870+0000 mon.a (mon.0) 1325 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2024-09-13T00:50:25.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:25 smithi088 bash[68960]: cluster 2024-09-13T00:50:24.189870+0000 mon.a (mon.0) 1325 : cluster [DBG] osdmap e822: 8 total, 8 up, 8 in 2024-09-13T00:50:26.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:26 smithi077 bash[80681]: cluster 2024-09-13T00:50:24.798842+0000 mgr.y (mgr.25304) 436 : cluster [DBG] pgmap v878: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 25 op/s 2024-09-13T00:50:26.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:26 smithi077 bash[80681]: cluster 2024-09-13T00:50:25.218744+0000 mon.a (mon.0) 1326 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2024-09-13T00:50:26.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:26 smithi077 bash[80681]: cluster 2024-09-13T00:50:26.076674+0000 mon.a (mon.0) 1327 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2024-09-13T00:50:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:26 smithi077 bash[83127]: cluster 2024-09-13T00:50:24.798842+0000 mgr.y (mgr.25304) 436 : cluster [DBG] pgmap v878: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 25 op/s 2024-09-13T00:50:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:26 smithi077 bash[83127]: cluster 2024-09-13T00:50:25.218744+0000 mon.a (mon.0) 1326 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2024-09-13T00:50:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:26 smithi077 bash[83127]: cluster 2024-09-13T00:50:26.076674+0000 mon.a (mon.0) 1327 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2024-09-13T00:50:26.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:26 smithi088 bash[68960]: cluster 2024-09-13T00:50:24.798842+0000 mgr.y (mgr.25304) 436 : cluster [DBG] pgmap v878: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 5.0 KiB/s wr, 25 op/s 2024-09-13T00:50:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:26 smithi088 bash[68960]: cluster 2024-09-13T00:50:25.218744+0000 mon.a (mon.0) 1326 : cluster [DBG] osdmap e823: 8 total, 8 up, 8 in 2024-09-13T00:50:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:26 smithi088 bash[68960]: cluster 2024-09-13T00:50:26.076674+0000 mon.a (mon.0) 1327 : cluster [DBG] osdmap e824: 8 total, 8 up, 8 in 2024-09-13T00:50:28.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:28 smithi088 bash[68960]: cluster 2024-09-13T00:50:26.800925+0000 mgr.y (mgr.25304) 437 : cluster [DBG] pgmap v881: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 5.0 KiB/s wr, 28 op/s 2024-09-13T00:50:28.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:28 smithi088 bash[68960]: cluster 2024-09-13T00:50:27.080496+0000 mon.a (mon.0) 1328 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2024-09-13T00:50:28.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:28 smithi077 bash[80681]: cluster 2024-09-13T00:50:26.800925+0000 mgr.y (mgr.25304) 437 : cluster [DBG] pgmap v881: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 5.0 KiB/s wr, 28 op/s 2024-09-13T00:50:28.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:28 smithi077 bash[80681]: cluster 2024-09-13T00:50:27.080496+0000 mon.a (mon.0) 1328 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2024-09-13T00:50:28.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:28 smithi077 bash[83127]: cluster 2024-09-13T00:50:26.800925+0000 mgr.y (mgr.25304) 437 : cluster [DBG] pgmap v881: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 5.0 KiB/s wr, 28 op/s 2024-09-13T00:50:28.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:28 smithi077 bash[83127]: cluster 2024-09-13T00:50:27.080496+0000 mon.a (mon.0) 1328 : cluster [DBG] osdmap e825: 8 total, 8 up, 8 in 2024-09-13T00:50:28.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:50:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:50:28] "GET /metrics HTTP/1.1" 200 82355 "" "Prometheus/2.43.0" 2024-09-13T00:50:29.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:29 smithi088 bash[68960]: cluster 2024-09-13T00:50:28.084581+0000 mon.a (mon.0) 1329 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2024-09-13T00:50:29.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:29 smithi088 bash[68960]: audit 2024-09-13T00:50:28.424999+0000 mon.a (mon.0) 1330 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:29.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:29 smithi077 bash[80681]: cluster 2024-09-13T00:50:28.084581+0000 mon.a (mon.0) 1329 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2024-09-13T00:50:29.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:29 smithi077 bash[80681]: audit 2024-09-13T00:50:28.424999+0000 mon.a (mon.0) 1330 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:29.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:29 smithi077 bash[83127]: cluster 2024-09-13T00:50:28.084581+0000 mon.a (mon.0) 1329 : cluster [DBG] osdmap e826: 8 total, 8 up, 8 in 2024-09-13T00:50:29.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:29 smithi077 bash[83127]: audit 2024-09-13T00:50:28.424999+0000 mon.a (mon.0) 1330 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:30.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:30 smithi088 bash[68960]: cluster 2024-09-13T00:50:28.802875+0000 mgr.y (mgr.25304) 438 : cluster [DBG] pgmap v884: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 255 B/s wr, 3 op/s 2024-09-13T00:50:30.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:30 smithi088 bash[68960]: cluster 2024-09-13T00:50:29.107528+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2024-09-13T00:50:30.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:30 smithi077 bash[80681]: cluster 2024-09-13T00:50:28.802875+0000 mgr.y (mgr.25304) 438 : cluster [DBG] pgmap v884: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 255 B/s wr, 3 op/s 2024-09-13T00:50:30.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:30 smithi077 bash[80681]: cluster 2024-09-13T00:50:29.107528+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2024-09-13T00:50:30.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:30 smithi077 bash[83127]: cluster 2024-09-13T00:50:28.802875+0000 mgr.y (mgr.25304) 438 : cluster [DBG] pgmap v884: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.0 KiB/s rd, 255 B/s wr, 3 op/s 2024-09-13T00:50:30.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:30 smithi077 bash[83127]: cluster 2024-09-13T00:50:29.107528+0000 mon.a (mon.0) 1331 : cluster [DBG] osdmap e827: 8 total, 8 up, 8 in 2024-09-13T00:50:31.294 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] TestLibRBD.ConcurentOperations (24877 ms) 2024-09-13T00:50:31.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:31 smithi077 bash[80681]: cluster 2024-09-13T00:50:30.120104+0000 mon.a (mon.0) 1332 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2024-09-13T00:50:31.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:31 smithi077 bash[80681]: cluster 2024-09-13T00:50:31.078772+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2024-09-13T00:50:31.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:31 smithi077 bash[83127]: cluster 2024-09-13T00:50:30.120104+0000 mon.a (mon.0) 1332 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2024-09-13T00:50:31.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:31 smithi077 bash[83127]: cluster 2024-09-13T00:50:31.078772+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2024-09-13T00:50:31.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:31 smithi088 bash[68960]: cluster 2024-09-13T00:50:30.120104+0000 mon.a (mon.0) 1332 : cluster [DBG] osdmap e828: 8 total, 8 up, 8 in 2024-09-13T00:50:31.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:31 smithi088 bash[68960]: cluster 2024-09-13T00:50:31.078772+0000 mon.a (mon.0) 1333 : cluster [DBG] osdmap e829: 8 total, 8 up, 8 in 2024-09-13T00:50:32.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:32 smithi077 bash[80681]: cluster 2024-09-13T00:50:30.804083+0000 mgr.y (mgr.25304) 439 : cluster [DBG] pgmap v887: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 63 KiB/s rd, 4.0 KiB/s wr, 82 op/s 2024-09-13T00:50:32.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:32 smithi077 bash[83127]: cluster 2024-09-13T00:50:30.804083+0000 mgr.y (mgr.25304) 439 : cluster [DBG] pgmap v887: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 63 KiB/s rd, 4.0 KiB/s wr, 82 op/s 2024-09-13T00:50:32.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:32 smithi088 bash[68960]: cluster 2024-09-13T00:50:30.804083+0000 mgr.y (mgr.25304) 439 : cluster [DBG] pgmap v887: 713 pgs: 713 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 63 KiB/s rd, 4.0 KiB/s wr, 82 op/s 2024-09-13T00:50:32.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:50:32 smithi088 bash[61791]: ts=2024-09-13T00:50:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:50:33.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:33 smithi077 bash[80681]: cluster 2024-09-13T00:50:32.132526+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2024-09-13T00:50:33.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:33 smithi077 bash[83127]: cluster 2024-09-13T00:50:32.132526+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2024-09-13T00:50:33.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:33 smithi088 bash[68960]: cluster 2024-09-13T00:50:32.132526+0000 mon.a (mon.0) 1334 : cluster [DBG] osdmap e830: 8 total, 8 up, 8 in 2024-09-13T00:50:34.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:34 smithi077 bash[83127]: cluster 2024-09-13T00:50:32.805232+0000 mgr.y (mgr.25304) 440 : cluster [DBG] pgmap v890: 681 pgs: 681 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 511 B/s wr, 51 op/s 2024-09-13T00:50:34.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:34 smithi077 bash[83127]: cluster 2024-09-13T00:50:33.136373+0000 mon.a (mon.0) 1335 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2024-09-13T00:50:34.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:34 smithi077 bash[83127]: cluster 2024-09-13T00:50:34.138271+0000 mon.a (mon.0) 1336 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2024-09-13T00:50:34.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:34 smithi077 bash[80681]: cluster 2024-09-13T00:50:32.805232+0000 mgr.y (mgr.25304) 440 : cluster [DBG] pgmap v890: 681 pgs: 681 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 511 B/s wr, 51 op/s 2024-09-13T00:50:34.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:34 smithi077 bash[80681]: cluster 2024-09-13T00:50:33.136373+0000 mon.a (mon.0) 1335 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2024-09-13T00:50:34.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:34 smithi077 bash[80681]: cluster 2024-09-13T00:50:34.138271+0000 mon.a (mon.0) 1336 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2024-09-13T00:50:34.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:34 smithi088 bash[68960]: cluster 2024-09-13T00:50:32.805232+0000 mgr.y (mgr.25304) 440 : cluster [DBG] pgmap v890: 681 pgs: 681 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 43 KiB/s rd, 511 B/s wr, 51 op/s 2024-09-13T00:50:34.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:34 smithi088 bash[68960]: cluster 2024-09-13T00:50:33.136373+0000 mon.a (mon.0) 1335 : cluster [DBG] osdmap e831: 8 total, 8 up, 8 in 2024-09-13T00:50:34.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:34 smithi088 bash[68960]: cluster 2024-09-13T00:50:34.138271+0000 mon.a (mon.0) 1336 : cluster [DBG] osdmap e832: 8 total, 8 up, 8 in 2024-09-13T00:50:36.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:36 smithi077 bash[80681]: cluster 2024-09-13T00:50:34.808785+0000 mgr.y (mgr.25304) 441 : cluster [DBG] pgmap v893: 617 pgs: 617 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:36.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:36 smithi077 bash[80681]: cluster 2024-09-13T00:50:35.140499+0000 mon.a (mon.0) 1337 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2024-09-13T00:50:36.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:36 smithi077 bash[80681]: cluster 2024-09-13T00:50:36.082868+0000 mon.a (mon.0) 1338 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2024-09-13T00:50:36.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:36 smithi077 bash[83127]: cluster 2024-09-13T00:50:34.808785+0000 mgr.y (mgr.25304) 441 : cluster [DBG] pgmap v893: 617 pgs: 617 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:36.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:36 smithi077 bash[83127]: cluster 2024-09-13T00:50:35.140499+0000 mon.a (mon.0) 1337 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2024-09-13T00:50:36.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:36 smithi077 bash[83127]: cluster 2024-09-13T00:50:36.082868+0000 mon.a (mon.0) 1338 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2024-09-13T00:50:36.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:36 smithi088 bash[68960]: cluster 2024-09-13T00:50:34.808785+0000 mgr.y (mgr.25304) 441 : cluster [DBG] pgmap v893: 617 pgs: 617 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:36.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:36 smithi088 bash[68960]: cluster 2024-09-13T00:50:35.140499+0000 mon.a (mon.0) 1337 : cluster [DBG] osdmap e833: 8 total, 8 up, 8 in 2024-09-13T00:50:36.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:36 smithi088 bash[68960]: cluster 2024-09-13T00:50:36.082868+0000 mon.a (mon.0) 1338 : cluster [DBG] osdmap e834: 8 total, 8 up, 8 in 2024-09-13T00:50:38.351 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:38 smithi088 bash[68960]: cluster 2024-09-13T00:50:36.810469+0000 mgr.y (mgr.25304) 442 : cluster [DBG] pgmap v896: 553 pgs: 553 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:38.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:38 smithi088 bash[68960]: cluster 2024-09-13T00:50:37.083991+0000 mon.a (mon.0) 1339 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2024-09-13T00:50:38.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:38 smithi077 bash[80681]: cluster 2024-09-13T00:50:36.810469+0000 mgr.y (mgr.25304) 442 : cluster [DBG] pgmap v896: 553 pgs: 553 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:38.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:38 smithi077 bash[80681]: cluster 2024-09-13T00:50:37.083991+0000 mon.a (mon.0) 1339 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2024-09-13T00:50:38.526 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:38 smithi077 bash[83127]: cluster 2024-09-13T00:50:36.810469+0000 mgr.y (mgr.25304) 442 : cluster [DBG] pgmap v896: 553 pgs: 553 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:38.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:38 smithi077 bash[83127]: cluster 2024-09-13T00:50:37.083991+0000 mon.a (mon.0) 1339 : cluster [DBG] osdmap e835: 8 total, 8 up, 8 in 2024-09-13T00:50:38.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:50:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:50:38] "GET /metrics HTTP/1.1" 200 82355 "" "Prometheus/2.43.0" 2024-09-13T00:50:39.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:39 smithi077 bash[80681]: cluster 2024-09-13T00:50:38.087427+0000 mon.a (mon.0) 1340 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2024-09-13T00:50:39.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:39 smithi077 bash[83127]: cluster 2024-09-13T00:50:38.087427+0000 mon.a (mon.0) 1340 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2024-09-13T00:50:39.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:39 smithi088 bash[68960]: cluster 2024-09-13T00:50:38.087427+0000 mon.a (mon.0) 1340 : cluster [DBG] osdmap e836: 8 total, 8 up, 8 in 2024-09-13T00:50:40.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:40 smithi077 bash[80681]: cluster 2024-09-13T00:50:38.812505+0000 mgr.y (mgr.25304) 443 : cluster [DBG] pgmap v899: 489 pgs: 489 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:40.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:40 smithi077 bash[80681]: cluster 2024-09-13T00:50:39.090279+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2024-09-13T00:50:40.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:40 smithi077 bash[80681]: cluster 2024-09-13T00:50:40.094809+0000 mon.a (mon.0) 1342 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2024-09-13T00:50:40.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:40 smithi077 bash[83127]: cluster 2024-09-13T00:50:38.812505+0000 mgr.y (mgr.25304) 443 : cluster [DBG] pgmap v899: 489 pgs: 489 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:40.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:40 smithi077 bash[83127]: cluster 2024-09-13T00:50:39.090279+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2024-09-13T00:50:40.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:40 smithi077 bash[83127]: cluster 2024-09-13T00:50:40.094809+0000 mon.a (mon.0) 1342 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2024-09-13T00:50:40.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:40 smithi088 bash[68960]: cluster 2024-09-13T00:50:38.812505+0000 mgr.y (mgr.25304) 443 : cluster [DBG] pgmap v899: 489 pgs: 489 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:40.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:40 smithi088 bash[68960]: cluster 2024-09-13T00:50:39.090279+0000 mon.a (mon.0) 1341 : cluster [DBG] osdmap e837: 8 total, 8 up, 8 in 2024-09-13T00:50:40.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:40 smithi088 bash[68960]: cluster 2024-09-13T00:50:40.094809+0000 mon.a (mon.0) 1342 : cluster [DBG] osdmap e838: 8 total, 8 up, 8 in 2024-09-13T00:50:42.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:42 smithi088 bash[68960]: cluster 2024-09-13T00:50:40.813412+0000 mgr.y (mgr.25304) 444 : cluster [DBG] pgmap v902: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:42.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:42 smithi088 bash[68960]: cluster 2024-09-13T00:50:41.095986+0000 mon.a (mon.0) 1343 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2024-09-13T00:50:42.352 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:50:42 smithi088 bash[61791]: ts=2024-09-13T00:50:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:50:42.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:42 smithi077 bash[80681]: cluster 2024-09-13T00:50:40.813412+0000 mgr.y (mgr.25304) 444 : cluster [DBG] pgmap v902: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:42.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:42 smithi077 bash[80681]: cluster 2024-09-13T00:50:41.095986+0000 mon.a (mon.0) 1343 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2024-09-13T00:50:42.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:42 smithi077 bash[83127]: cluster 2024-09-13T00:50:40.813412+0000 mgr.y (mgr.25304) 444 : cluster [DBG] pgmap v902: 425 pgs: 425 active+clean; 2.6 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:42.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:42 smithi077 bash[83127]: cluster 2024-09-13T00:50:41.095986+0000 mon.a (mon.0) 1343 : cluster [DBG] osdmap e839: 8 total, 8 up, 8 in 2024-09-13T00:50:43.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:43 smithi077 bash[83127]: cluster 2024-09-13T00:50:42.109211+0000 mon.a (mon.0) 1344 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2024-09-13T00:50:43.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:43 smithi077 bash[80681]: cluster 2024-09-13T00:50:42.109211+0000 mon.a (mon.0) 1344 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2024-09-13T00:50:43.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:43 smithi088 bash[68960]: cluster 2024-09-13T00:50:42.109211+0000 mon.a (mon.0) 1344 : cluster [DBG] osdmap e840: 8 total, 8 up, 8 in 2024-09-13T00:50:44.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:44 smithi077 bash[80681]: cluster 2024-09-13T00:50:42.814434+0000 mgr.y (mgr.25304) 445 : cluster [DBG] pgmap v905: 361 pgs: 361 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:44.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:44 smithi077 bash[80681]: cluster 2024-09-13T00:50:43.122903+0000 mon.a (mon.0) 1345 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2024-09-13T00:50:44.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:44 smithi077 bash[80681]: audit 2024-09-13T00:50:43.425148+0000 mon.a (mon.0) 1346 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:44.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:44 smithi077 bash[83127]: cluster 2024-09-13T00:50:42.814434+0000 mgr.y (mgr.25304) 445 : cluster [DBG] pgmap v905: 361 pgs: 361 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:44.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:44 smithi077 bash[83127]: cluster 2024-09-13T00:50:43.122903+0000 mon.a (mon.0) 1345 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2024-09-13T00:50:44.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:44 smithi077 bash[83127]: audit 2024-09-13T00:50:43.425148+0000 mon.a (mon.0) 1346 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:44.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:44 smithi088 bash[68960]: cluster 2024-09-13T00:50:42.814434+0000 mgr.y (mgr.25304) 445 : cluster [DBG] pgmap v905: 361 pgs: 361 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:44.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:44 smithi088 bash[68960]: cluster 2024-09-13T00:50:43.122903+0000 mon.a (mon.0) 1345 : cluster [DBG] osdmap e841: 8 total, 8 up, 8 in 2024-09-13T00:50:44.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:44 smithi088 bash[68960]: audit 2024-09-13T00:50:43.425148+0000 mon.a (mon.0) 1346 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:45.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:45 smithi077 bash[83127]: cluster 2024-09-13T00:50:44.121314+0000 mon.a (mon.0) 1347 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2024-09-13T00:50:45.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:45 smithi077 bash[83127]: cluster 2024-09-13T00:50:45.125679+0000 mon.a (mon.0) 1348 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2024-09-13T00:50:45.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:45 smithi077 bash[80681]: cluster 2024-09-13T00:50:44.121314+0000 mon.a (mon.0) 1347 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2024-09-13T00:50:45.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:45 smithi077 bash[80681]: cluster 2024-09-13T00:50:45.125679+0000 mon.a (mon.0) 1348 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2024-09-13T00:50:45.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:45 smithi088 bash[68960]: cluster 2024-09-13T00:50:44.121314+0000 mon.a (mon.0) 1347 : cluster [DBG] osdmap e842: 8 total, 8 up, 8 in 2024-09-13T00:50:45.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:45 smithi088 bash[68960]: cluster 2024-09-13T00:50:45.125679+0000 mon.a (mon.0) 1348 : cluster [DBG] osdmap e843: 8 total, 8 up, 8 in 2024-09-13T00:50:46.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:46 smithi077 bash[80681]: cluster 2024-09-13T00:50:44.816380+0000 mgr.y (mgr.25304) 446 : cluster [DBG] pgmap v908: 297 pgs: 297 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:46.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:46 smithi077 bash[80681]: cluster 2024-09-13T00:50:46.126780+0000 mon.a (mon.0) 1349 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2024-09-13T00:50:46.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:46 smithi077 bash[83127]: cluster 2024-09-13T00:50:44.816380+0000 mgr.y (mgr.25304) 446 : cluster [DBG] pgmap v908: 297 pgs: 297 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:46.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:46 smithi077 bash[83127]: cluster 2024-09-13T00:50:46.126780+0000 mon.a (mon.0) 1349 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2024-09-13T00:50:46.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:46 smithi088 bash[68960]: cluster 2024-09-13T00:50:44.816380+0000 mgr.y (mgr.25304) 446 : cluster [DBG] pgmap v908: 297 pgs: 297 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:46.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:46 smithi088 bash[68960]: cluster 2024-09-13T00:50:46.126780+0000 mon.a (mon.0) 1349 : cluster [DBG] osdmap e844: 8 total, 8 up, 8 in 2024-09-13T00:50:48.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:48 smithi077 bash[80681]: cluster 2024-09-13T00:50:46.817490+0000 mgr.y (mgr.25304) 447 : cluster [DBG] pgmap v911: 233 pgs: 233 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:48.526 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:48 smithi077 bash[80681]: cluster 2024-09-13T00:50:47.130620+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2024-09-13T00:50:48.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:48 smithi077 bash[83127]: cluster 2024-09-13T00:50:46.817490+0000 mgr.y (mgr.25304) 447 : cluster [DBG] pgmap v911: 233 pgs: 233 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:48.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:48 smithi077 bash[83127]: cluster 2024-09-13T00:50:47.130620+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2024-09-13T00:50:48.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:48 smithi088 bash[68960]: cluster 2024-09-13T00:50:46.817490+0000 mgr.y (mgr.25304) 447 : cluster [DBG] pgmap v911: 233 pgs: 233 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:48.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:48 smithi088 bash[68960]: cluster 2024-09-13T00:50:47.130620+0000 mon.a (mon.0) 1350 : cluster [DBG] osdmap e845: 8 total, 8 up, 8 in 2024-09-13T00:50:48.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:50:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:50:48] "GET /metrics HTTP/1.1" 200 53470 "" "Prometheus/2.43.0" 2024-09-13T00:50:49.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:49 smithi077 bash[80681]: cluster 2024-09-13T00:50:48.136426+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2024-09-13T00:50:49.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:49 smithi077 bash[80681]: cluster 2024-09-13T00:50:49.140624+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2024-09-13T00:50:49.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:49 smithi077 bash[83127]: cluster 2024-09-13T00:50:48.136426+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2024-09-13T00:50:49.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:49 smithi077 bash[83127]: cluster 2024-09-13T00:50:49.140624+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2024-09-13T00:50:49.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:49 smithi088 bash[68960]: cluster 2024-09-13T00:50:48.136426+0000 mon.a (mon.0) 1351 : cluster [DBG] osdmap e846: 8 total, 8 up, 8 in 2024-09-13T00:50:49.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:49 smithi088 bash[68960]: cluster 2024-09-13T00:50:49.140624+0000 mon.a (mon.0) 1352 : cluster [DBG] osdmap e847: 8 total, 8 up, 8 in 2024-09-13T00:50:50.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:50 smithi077 bash[80681]: cluster 2024-09-13T00:50:48.818551+0000 mgr.y (mgr.25304) 448 : cluster [DBG] pgmap v914: 169 pgs: 169 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:50.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:50 smithi077 bash[80681]: cluster 2024-09-13T00:50:50.141641+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2024-09-13T00:50:50.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:50 smithi077 bash[83127]: cluster 2024-09-13T00:50:48.818551+0000 mgr.y (mgr.25304) 448 : cluster [DBG] pgmap v914: 169 pgs: 169 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:50.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:50 smithi077 bash[83127]: cluster 2024-09-13T00:50:50.141641+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2024-09-13T00:50:50.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:50 smithi088 bash[68960]: cluster 2024-09-13T00:50:48.818551+0000 mgr.y (mgr.25304) 448 : cluster [DBG] pgmap v914: 169 pgs: 169 active+clean; 587 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:50.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:50 smithi088 bash[68960]: cluster 2024-09-13T00:50:50.141641+0000 mon.a (mon.0) 1353 : cluster [DBG] osdmap e848: 8 total, 8 up, 8 in 2024-09-13T00:50:51.145 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 160 tests from TestLibRBD (376807 ms total) 2024-09-13T00:50:51.145 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:50:51.145 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 11 tests from DiffIterateTest/0, where TypeParam = DiffIterateParams 2024-09-13T00:50:51.198 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/0.DiffIterate 2024-09-13T00:50:52.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:52 smithi077 bash[80681]: cluster 2024-09-13T00:50:50.819298+0000 mgr.y (mgr.25304) 449 : cluster [DBG] pgmap v917: 105 pgs: 105 active+clean; 586 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:52.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:52 smithi077 bash[80681]: cluster 2024-09-13T00:50:51.141203+0000 mon.a (mon.0) 1354 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2024-09-13T00:50:52.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:52 smithi077 bash[83127]: cluster 2024-09-13T00:50:50.819298+0000 mgr.y (mgr.25304) 449 : cluster [DBG] pgmap v917: 105 pgs: 105 active+clean; 586 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:52.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:52 smithi077 bash[83127]: cluster 2024-09-13T00:50:51.141203+0000 mon.a (mon.0) 1354 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2024-09-13T00:50:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:52 smithi088 bash[68960]: cluster 2024-09-13T00:50:50.819298+0000 mgr.y (mgr.25304) 449 : cluster [DBG] pgmap v917: 105 pgs: 105 active+clean; 586 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:52 smithi088 bash[68960]: cluster 2024-09-13T00:50:51.141203+0000 mon.a (mon.0) 1354 : cluster [DBG] osdmap e849: 8 total, 8 up, 8 in 2024-09-13T00:50:52.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:50:52 smithi088 bash[61791]: ts=2024-09-13T00:50:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:50:53.179 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:50:53.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:53 smithi077 bash[83127]: cluster 2024-09-13T00:50:52.173431+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2024-09-13T00:50:53.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:53 smithi077 bash[83127]: audit 2024-09-13T00:50:52.176341+0000 mon.a (mon.0) 1356 : audit [INF] from='client.? 172.21.15.77:0/4239320093' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:50:53.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:53 smithi077 bash[80681]: cluster 2024-09-13T00:50:52.173431+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2024-09-13T00:50:53.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:53 smithi077 bash[80681]: audit 2024-09-13T00:50:52.176341+0000 mon.a (mon.0) 1356 : audit [INF] from='client.? 172.21.15.77:0/4239320093' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:50:53.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:53 smithi088 bash[68960]: cluster 2024-09-13T00:50:52.173431+0000 mon.a (mon.0) 1355 : cluster [DBG] osdmap e850: 8 total, 8 up, 8 in 2024-09-13T00:50:53.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:53 smithi088 bash[68960]: audit 2024-09-13T00:50:52.176341+0000 mon.a (mon.0) 1356 : audit [INF] from='client.? 172.21.15.77:0/4239320093' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:50:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:54 smithi077 bash[80681]: cluster 2024-09-13T00:50:52.819993+0000 mgr.y (mgr.25304) 450 : cluster [DBG] pgmap v920: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:54 smithi077 bash[80681]: cluster 2024-09-13T00:50:53.167827+0000 mon.a (mon.0) 1357 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:50:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:54 smithi077 bash[80681]: audit 2024-09-13T00:50:53.176274+0000 mon.a (mon.0) 1358 : audit [INF] from='client.? 172.21.15.77:0/4239320093' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:50:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:54 smithi077 bash[80681]: cluster 2024-09-13T00:50:53.176843+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2024-09-13T00:50:54.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:54 smithi077 bash[83127]: cluster 2024-09-13T00:50:52.819993+0000 mgr.y (mgr.25304) 450 : cluster [DBG] pgmap v920: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:54.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:54 smithi077 bash[83127]: cluster 2024-09-13T00:50:53.167827+0000 mon.a (mon.0) 1357 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:50:54.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:54 smithi077 bash[83127]: audit 2024-09-13T00:50:53.176274+0000 mon.a (mon.0) 1358 : audit [INF] from='client.? 172.21.15.77:0/4239320093' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:50:54.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:54 smithi077 bash[83127]: cluster 2024-09-13T00:50:53.176843+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2024-09-13T00:50:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:54 smithi088 bash[68960]: cluster 2024-09-13T00:50:52.819993+0000 mgr.y (mgr.25304) 450 : cluster [DBG] pgmap v920: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:54 smithi088 bash[68960]: cluster 2024-09-13T00:50:53.167827+0000 mon.a (mon.0) 1357 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-13T00:50:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:54 smithi088 bash[68960]: audit 2024-09-13T00:50:53.176274+0000 mon.a (mon.0) 1358 : audit [INF] from='client.? 172.21.15.77:0/4239320093' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:50:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:54 smithi088 bash[68960]: cluster 2024-09-13T00:50:53.176843+0000 mon.a (mon.0) 1359 : cluster [DBG] osdmap e851: 8 total, 8 up, 8 in 2024-09-13T00:50:55.272 INFO:tasks.workunit.client.0.smithi077.stdout: wrote [1046744~13920,6956516~82249,9332706~1689,9585592~34620,11029193~43834,14341206~21784,15114831~53843,17622935~75032,19385020~87399,19811569~91017] 2024-09-13T00:50:55.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:55 smithi077 bash[83127]: cluster 2024-09-13T00:50:54.197152+0000 mon.a (mon.0) 1360 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2024-09-13T00:50:55.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:55 smithi077 bash[80681]: cluster 2024-09-13T00:50:54.197152+0000 mon.a (mon.0) 1360 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2024-09-13T00:50:55.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:55 smithi088 bash[68960]: cluster 2024-09-13T00:50:54.197152+0000 mon.a (mon.0) 1360 : cluster [DBG] osdmap e852: 8 total, 8 up, 8 in 2024-09-13T00:50:56.247 INFO:tasks.workunit.client.0.smithi077.stdout: wrote [1261763~26407,4695329~75350,7640190~1348,13965326~94149,14767024~1756,15320321~12480,15558403~97513,16413342~44787,17191342~72651,19186114~3441] 2024-09-13T00:50:56.263 INFO:tasks.workunit.client.0.smithi077.stdout: diff was [1060664~227506,4695329~75350,7038765~602773,13965326~94149,14767024~1756,15168674~1289455,17191342~72651,19186114~3441] 2024-09-13T00:50:56.272 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/0.DiffIterate (5074 ms) 2024-09-13T00:50:56.272 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDiscard 2024-09-13T00:50:56.272 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:50:56.333 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 0~256 2024-09-13T00:50:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:56 smithi077 bash[80681]: cluster 2024-09-13T00:50:54.821001+0000 mgr.y (mgr.25304) 451 : cluster [DBG] pgmap v923: 105 pgs: 8 creating+peering, 4 unknown, 93 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:56 smithi077 bash[80681]: cluster 2024-09-13T00:50:55.193733+0000 mon.a (mon.0) 1361 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:50:56.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:56 smithi077 bash[80681]: cluster 2024-09-13T00:50:55.212751+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2024-09-13T00:50:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:56 smithi077 bash[83127]: cluster 2024-09-13T00:50:54.821001+0000 mgr.y (mgr.25304) 451 : cluster [DBG] pgmap v923: 105 pgs: 8 creating+peering, 4 unknown, 93 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:56 smithi077 bash[83127]: cluster 2024-09-13T00:50:55.193733+0000 mon.a (mon.0) 1361 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:50:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:56 smithi077 bash[83127]: cluster 2024-09-13T00:50:55.212751+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2024-09-13T00:50:56.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:56 smithi088 bash[68960]: cluster 2024-09-13T00:50:54.821001+0000 mgr.y (mgr.25304) 451 : cluster [DBG] pgmap v923: 105 pgs: 8 creating+peering, 4 unknown, 93 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:50:56.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:56 smithi088 bash[68960]: cluster 2024-09-13T00:50:55.193733+0000 mon.a (mon.0) 1361 : cluster [INF] Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-13T00:50:56.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:56 smithi088 bash[68960]: cluster 2024-09-13T00:50:55.212751+0000 mon.a (mon.0) 1362 : cluster [DBG] osdmap e853: 8 total, 8 up, 8 in 2024-09-13T00:50:57.265 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 0~256 2024-09-13T00:50:57.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:57 smithi077 bash[80681]: cluster 2024-09-13T00:50:56.222652+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2024-09-13T00:50:57.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:57 smithi077 bash[83127]: cluster 2024-09-13T00:50:56.222652+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2024-09-13T00:50:57.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:57 smithi088 bash[68960]: cluster 2024-09-13T00:50:56.222652+0000 mon.a (mon.0) 1363 : cluster [DBG] osdmap e854: 8 total, 8 up, 8 in 2024-09-13T00:50:58.269 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 0~256 2024-09-13T00:50:58.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:58 smithi077 bash[80681]: cluster 2024-09-13T00:50:56.821798+0000 mgr.y (mgr.25304) 452 : cluster [DBG] pgmap v926: 105 pgs: 8 creating+peering, 97 active+clean; 3.1 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 20 KiB/s wr, 1 op/s 2024-09-13T00:50:58.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:58 smithi077 bash[80681]: cluster 2024-09-13T00:50:57.232823+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2024-09-13T00:50:58.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:58 smithi077 bash[80681]: audit 2024-09-13T00:50:57.606609+0000 mon.a (mon.0) 1365 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:50:58.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:58 smithi077 bash[80681]: audit 2024-09-13T00:50:57.974652+0000 mon.a (mon.0) 1366 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:50:58.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:58 smithi077 bash[80681]: audit 2024-09-13T00:50:57.980499+0000 mon.a (mon.0) 1367 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:50:58.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:58 smithi077 bash[80681]: audit 2024-09-13T00:50:57.986774+0000 mon.a (mon.0) 1368 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:50:58.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:58 smithi077 bash[80681]: audit 2024-09-13T00:50:57.995493+0000 mon.a (mon.0) 1369 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:50:58.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:58 smithi077 bash[83127]: cluster 2024-09-13T00:50:56.821798+0000 mgr.y (mgr.25304) 452 : cluster [DBG] pgmap v926: 105 pgs: 8 creating+peering, 97 active+clean; 3.1 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 20 KiB/s wr, 1 op/s 2024-09-13T00:50:58.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:58 smithi077 bash[83127]: cluster 2024-09-13T00:50:57.232823+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2024-09-13T00:50:58.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:58 smithi077 bash[83127]: audit 2024-09-13T00:50:57.606609+0000 mon.a (mon.0) 1365 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:50:58.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:58 smithi077 bash[83127]: audit 2024-09-13T00:50:57.974652+0000 mon.a (mon.0) 1366 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:50:58.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:58 smithi077 bash[83127]: audit 2024-09-13T00:50:57.980499+0000 mon.a (mon.0) 1367 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:50:58.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:58 smithi077 bash[83127]: audit 2024-09-13T00:50:57.986774+0000 mon.a (mon.0) 1368 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:50:58.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:58 smithi077 bash[83127]: audit 2024-09-13T00:50:57.995493+0000 mon.a (mon.0) 1369 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:50:58.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:58 smithi088 bash[68960]: cluster 2024-09-13T00:50:56.821798+0000 mgr.y (mgr.25304) 452 : cluster [DBG] pgmap v926: 105 pgs: 8 creating+peering, 97 active+clean; 3.1 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 20 KiB/s wr, 1 op/s 2024-09-13T00:50:58.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:58 smithi088 bash[68960]: cluster 2024-09-13T00:50:57.232823+0000 mon.a (mon.0) 1364 : cluster [DBG] osdmap e855: 8 total, 8 up, 8 in 2024-09-13T00:50:58.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:58 smithi088 bash[68960]: audit 2024-09-13T00:50:57.606609+0000 mon.a (mon.0) 1365 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:50:58.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:58 smithi088 bash[68960]: audit 2024-09-13T00:50:57.974652+0000 mon.a (mon.0) 1366 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:50:58.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:58 smithi088 bash[68960]: audit 2024-09-13T00:50:57.980499+0000 mon.a (mon.0) 1367 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:50:58.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:58 smithi088 bash[68960]: audit 2024-09-13T00:50:57.986774+0000 mon.a (mon.0) 1368 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:50:58.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:58 smithi088 bash[68960]: audit 2024-09-13T00:50:57.995493+0000 mon.a (mon.0) 1369 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:50:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:50:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:50:58] "GET /metrics HTTP/1.1" 200 36676 "" "Prometheus/2.43.0" 2024-09-13T00:50:59.282 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 0~256 2024-09-13T00:50:59.290 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/0.DiffIterateDiscard (3018 ms) 2024-09-13T00:50:59.290 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStress 2024-09-13T00:50:59.290 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:50:59.419 INFO:tasks.workunit.client.0.smithi077.stdout: i=0 exists [25414253~303245,44644623~7595311,66532596~3592128,74618545~2004204,132712348~6401017,214097157~6279146,230698020~664117,245374890~5017042,377438429~8014718,401653630~3766966] wrote [25414253~303245,44644623~7595311,66532596~3592128,74618545~2004204,132712348~6401017,214097157~6279146,230698020~664117,245374890~5017042,377438429~8014718,401653630~3766966] 2024-09-13T00:50:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:59 smithi077 bash[83127]: cluster 2024-09-13T00:50:58.243492+0000 mon.a (mon.0) 1370 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2024-09-13T00:50:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:50:59 smithi077 bash[83127]: audit 2024-09-13T00:50:58.425603+0000 mon.a (mon.0) 1371 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:59.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:59 smithi077 bash[80681]: cluster 2024-09-13T00:50:58.243492+0000 mon.a (mon.0) 1370 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2024-09-13T00:50:59.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:50:59 smithi077 bash[80681]: audit 2024-09-13T00:50:58.425603+0000 mon.a (mon.0) 1371 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:50:59.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:59 smithi088 bash[68960]: cluster 2024-09-13T00:50:58.243492+0000 mon.a (mon.0) 1370 : cluster [DBG] osdmap e856: 8 total, 8 up, 8 in 2024-09-13T00:50:59.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:50:59 smithi088 bash[68960]: audit 2024-09-13T00:50:58.425603+0000 mon.a (mon.0) 1371 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:00.378 INFO:tasks.workunit.client.0.smithi077.stdout: i=1 exists [25414253~303245,44644623~7595311,66532596~3592128,74618545~2004204,120022934~6051327,132712348~6401017,207727920~4626596,214097157~6279146,230698020~664117,245374890~5017042,277817514~8150813,350547027~2260147,365523275~6858519,377438429~8014718,393516028~4657160,401653630~3766966,408258776~4450348] wrote [120022934~6051327,207727920~4626596,214186959~425193,277817514~8150813,350547027~2260147,365523275~6858519,393516028~4657160,408258776~4450348] 2024-09-13T00:51:00.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:00 smithi077 bash[80681]: cluster 2024-09-13T00:50:58.822597+0000 mgr.y (mgr.25304) 453 : cluster [DBG] pgmap v929: 105 pgs: 105 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 155 KiB/s wr, 27 op/s 2024-09-13T00:51:00.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:00 smithi077 bash[80681]: cluster 2024-09-13T00:50:59.252999+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2024-09-13T00:51:00.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:00 smithi077 bash[83127]: cluster 2024-09-13T00:50:58.822597+0000 mgr.y (mgr.25304) 453 : cluster [DBG] pgmap v929: 105 pgs: 105 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 155 KiB/s wr, 27 op/s 2024-09-13T00:51:00.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:00 smithi077 bash[83127]: cluster 2024-09-13T00:50:59.252999+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2024-09-13T00:51:00.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:00 smithi088 bash[68960]: cluster 2024-09-13T00:50:58.822597+0000 mgr.y (mgr.25304) 453 : cluster [DBG] pgmap v929: 105 pgs: 105 active+clean; 11 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 155 KiB/s wr, 27 op/s 2024-09-13T00:51:00.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:00 smithi088 bash[68960]: cluster 2024-09-13T00:50:59.252999+0000 mon.a (mon.0) 1372 : cluster [DBG] osdmap e857: 8 total, 8 up, 8 in 2024-09-13T00:51:01.373 INFO:tasks.workunit.client.0.smithi077.stdout: i=2 exists [11835209~1716033,25414253~303245,27215137~984633,33004547~1877145,44644623~7595311,62780271~112299,66532596~3592128,74618545~2004204,120022934~8804768,132712348~6401017,167226701~1958761,177616439~6534770,207727920~4626596,214097157~6279146,230698020~664117,245374890~5017042,275875051~10093276,288050963~868686,312922432~4810361,350547027~2260147,365523275~6858519,377438429~8014718,393516028~4657160,401653630~3766966,408258776~4450348] wrote [11835209~1716033,27215137~984633,33004547~1877145,62780271~112299,121899613~6928089,167226701~1958761,177616439~6534770,275875051~7619440,288050963~868686,312922432~4810361] 2024-09-13T00:51:01.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:01 smithi077 bash[80681]: cluster 2024-09-13T00:51:00.254832+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2024-09-13T00:51:01.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:01 smithi077 bash[83127]: cluster 2024-09-13T00:51:00.254832+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2024-09-13T00:51:01.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:01 smithi088 bash[68960]: cluster 2024-09-13T00:51:00.254832+0000 mon.a (mon.0) 1373 : cluster [DBG] osdmap e858: 8 total, 8 up, 8 in 2024-09-13T00:51:02.355 INFO:tasks.workunit.client.0.smithi077.stdout: i=3 exists [11835209~1716033,25414253~303245,27215137~984633,33004547~1877145,44644623~7595311,62780271~112299,66532596~3592128,74618545~2004204,104204950~2459588,120022934~8804768,132712348~6401017,167226701~1958761,177616439~6534770,207727920~4626596,214097157~6279146,230698020~664117,245374890~5017042,268623601~2612185,272690745~1385497,275274515~10693812,288050963~868686,312922432~4810361,321663886~5211040,343111568~2086045,350547027~2260147,363850683~866908,365523275~8337052,377438429~8928185,393516028~4657160,401653630~3766966,408258776~4450348] wrote [104204950~2459588,268623601~2612185,272690745~1385497,275274515~3080302,321663886~5211040,343111568~2086045,363850683~866908,366062574~7797753,385267053~1099561,409582788~931046] 2024-09-13T00:51:02.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:51:02 smithi088 bash[61791]: ts=2024-09-13T00:51:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:51:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:02 smithi088 bash[68960]: cluster 2024-09-13T00:51:00.823267+0000 mgr.y (mgr.25304) 454 : cluster [DBG] pgmap v932: 105 pgs: 105 active+clean; 36 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 3.7 MiB/s wr, 110 op/s 2024-09-13T00:51:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:02 smithi088 bash[68960]: cluster 2024-09-13T00:51:01.279009+0000 mon.a (mon.0) 1374 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2024-09-13T00:51:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:02 smithi088 bash[68960]: audit 2024-09-13T00:51:01.874143+0000 mon.a (mon.0) 1375 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]: dispatch 2024-09-13T00:51:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:02 smithi088 bash[68960]: audit 2024-09-13T00:51:01.874671+0000 mon.a (mon.0) 1376 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]: dispatch 2024-09-13T00:51:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:02 smithi088 bash[68960]: audit 2024-09-13T00:51:01.875043+0000 mon.a (mon.0) 1377 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]: dispatch 2024-09-13T00:51:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:02 smithi088 bash[68960]: audit 2024-09-13T00:51:01.875391+0000 mon.a (mon.0) 1378 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1f", "id": [5, 1]}]: dispatch 2024-09-13T00:51:02.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:02 smithi077 bash[80681]: cluster 2024-09-13T00:51:00.823267+0000 mgr.y (mgr.25304) 454 : cluster [DBG] pgmap v932: 105 pgs: 105 active+clean; 36 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 3.7 MiB/s wr, 110 op/s 2024-09-13T00:51:02.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:02 smithi077 bash[80681]: cluster 2024-09-13T00:51:01.279009+0000 mon.a (mon.0) 1374 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2024-09-13T00:51:02.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:02 smithi077 bash[80681]: audit 2024-09-13T00:51:01.874143+0000 mon.a (mon.0) 1375 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]: dispatch 2024-09-13T00:51:02.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:02 smithi077 bash[80681]: audit 2024-09-13T00:51:01.874671+0000 mon.a (mon.0) 1376 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]: dispatch 2024-09-13T00:51:02.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:02 smithi077 bash[80681]: audit 2024-09-13T00:51:01.875043+0000 mon.a (mon.0) 1377 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]: dispatch 2024-09-13T00:51:02.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:02 smithi077 bash[80681]: audit 2024-09-13T00:51:01.875391+0000 mon.a (mon.0) 1378 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1f", "id": [5, 1]}]: dispatch 2024-09-13T00:51:02.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:02 smithi077 bash[83127]: cluster 2024-09-13T00:51:00.823267+0000 mgr.y (mgr.25304) 454 : cluster [DBG] pgmap v932: 105 pgs: 105 active+clean; 36 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 3.7 MiB/s wr, 110 op/s 2024-09-13T00:51:02.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:02 smithi077 bash[83127]: cluster 2024-09-13T00:51:01.279009+0000 mon.a (mon.0) 1374 : cluster [DBG] osdmap e859: 8 total, 8 up, 8 in 2024-09-13T00:51:02.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:02 smithi077 bash[83127]: audit 2024-09-13T00:51:01.874143+0000 mon.a (mon.0) 1375 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]: dispatch 2024-09-13T00:51:02.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:02 smithi077 bash[83127]: audit 2024-09-13T00:51:01.874671+0000 mon.a (mon.0) 1376 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]: dispatch 2024-09-13T00:51:02.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:02 smithi077 bash[83127]: audit 2024-09-13T00:51:01.875043+0000 mon.a (mon.0) 1377 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]: dispatch 2024-09-13T00:51:02.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:02 smithi077 bash[83127]: audit 2024-09-13T00:51:01.875391+0000 mon.a (mon.0) 1378 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1f", "id": [5, 1]}]: dispatch 2024-09-13T00:51:03.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:03 smithi088 bash[68960]: audit 2024-09-13T00:51:02.283663+0000 mon.a (mon.0) 1379 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-09-13T00:51:03.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:03 smithi088 bash[68960]: audit 2024-09-13T00:51:02.283895+0000 mon.a (mon.0) 1380 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]': finished 2024-09-13T00:51:03.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:03 smithi088 bash[68960]: audit 2024-09-13T00:51:02.284105+0000 mon.a (mon.0) 1381 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]': finished 2024-09-13T00:51:03.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:03 smithi088 bash[68960]: audit 2024-09-13T00:51:02.284254+0000 mon.a (mon.0) 1382 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1f", "id": [5, 1]}]': finished 2024-09-13T00:51:03.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:03 smithi088 bash[68960]: cluster 2024-09-13T00:51:02.284395+0000 mon.a (mon.0) 1383 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2024-09-13T00:51:03.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:03 smithi088 bash[68960]: cluster 2024-09-13T00:51:03.276562+0000 mon.a (mon.0) 1384 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2024-09-13T00:51:03.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:03 smithi077 bash[80681]: audit 2024-09-13T00:51:02.283663+0000 mon.a (mon.0) 1379 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-09-13T00:51:03.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:03 smithi077 bash[80681]: audit 2024-09-13T00:51:02.283895+0000 mon.a (mon.0) 1380 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]': finished 2024-09-13T00:51:03.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:03 smithi077 bash[80681]: audit 2024-09-13T00:51:02.284105+0000 mon.a (mon.0) 1381 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]': finished 2024-09-13T00:51:03.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:03 smithi077 bash[80681]: audit 2024-09-13T00:51:02.284254+0000 mon.a (mon.0) 1382 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1f", "id": [5, 1]}]': finished 2024-09-13T00:51:03.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:03 smithi077 bash[80681]: cluster 2024-09-13T00:51:02.284395+0000 mon.a (mon.0) 1383 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2024-09-13T00:51:03.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:03 smithi077 bash[80681]: cluster 2024-09-13T00:51:03.276562+0000 mon.a (mon.0) 1384 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2024-09-13T00:51:03.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:03 smithi077 bash[83127]: audit 2024-09-13T00:51:02.283663+0000 mon.a (mon.0) 1379 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-09-13T00:51:03.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:03 smithi077 bash[83127]: audit 2024-09-13T00:51:02.283895+0000 mon.a (mon.0) 1380 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]': finished 2024-09-13T00:51:03.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:03 smithi077 bash[83127]: audit 2024-09-13T00:51:02.284105+0000 mon.a (mon.0) 1381 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]': finished 2024-09-13T00:51:03.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:03 smithi077 bash[83127]: audit 2024-09-13T00:51:02.284254+0000 mon.a (mon.0) 1382 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1f", "id": [5, 1]}]': finished 2024-09-13T00:51:03.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:03 smithi077 bash[83127]: cluster 2024-09-13T00:51:02.284395+0000 mon.a (mon.0) 1383 : cluster [DBG] osdmap e860: 8 total, 8 up, 8 in 2024-09-13T00:51:03.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:03 smithi077 bash[83127]: cluster 2024-09-13T00:51:03.276562+0000 mon.a (mon.0) 1384 : cluster [DBG] osdmap e861: 8 total, 8 up, 8 in 2024-09-13T00:51:04.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:04 smithi088 bash[68960]: cluster 2024-09-13T00:51:02.823973+0000 mgr.y (mgr.25304) 455 : cluster [DBG] pgmap v935: 105 pgs: 105 active+clean; 56 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 56 KiB/s rd, 9.1 MiB/s wr, 88 op/s 2024-09-13T00:51:04.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:04 smithi088 bash[68960]: audit 2024-09-13T00:51:03.558187+0000 mon.a (mon.0) 1385 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:04.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:04 smithi088 bash[68960]: audit 2024-09-13T00:51:03.565602+0000 mon.a (mon.0) 1386 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:04.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:04 smithi088 bash[68960]: audit 2024-09-13T00:51:03.733875+0000 mon.a (mon.0) 1387 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:04.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:04 smithi088 bash[68960]: audit 2024-09-13T00:51:03.844836+0000 mon.a (mon.0) 1388 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:04.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:04 smithi088 bash[68960]: audit 2024-09-13T00:51:03.851635+0000 mon.a (mon.0) 1389 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:04.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:04 smithi088 bash[68960]: audit 2024-09-13T00:51:04.290901+0000 mon.a (mon.0) 1390 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:51:04.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:04 smithi088 bash[68960]: audit 2024-09-13T00:51:04.292428+0000 mon.a (mon.0) 1391 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:51:04.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:04 smithi077 bash[80681]: cluster 2024-09-13T00:51:02.823973+0000 mgr.y (mgr.25304) 455 : cluster [DBG] pgmap v935: 105 pgs: 105 active+clean; 56 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 56 KiB/s rd, 9.1 MiB/s wr, 88 op/s 2024-09-13T00:51:04.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:04 smithi077 bash[80681]: audit 2024-09-13T00:51:03.558187+0000 mon.a (mon.0) 1385 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:04.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:04 smithi077 bash[80681]: audit 2024-09-13T00:51:03.565602+0000 mon.a (mon.0) 1386 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:04.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:04 smithi077 bash[80681]: audit 2024-09-13T00:51:03.733875+0000 mon.a (mon.0) 1387 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:04.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:04 smithi077 bash[80681]: audit 2024-09-13T00:51:03.844836+0000 mon.a (mon.0) 1388 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:04.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:04 smithi077 bash[80681]: audit 2024-09-13T00:51:03.851635+0000 mon.a (mon.0) 1389 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:04.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:04 smithi077 bash[80681]: audit 2024-09-13T00:51:04.290901+0000 mon.a (mon.0) 1390 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:51:04.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:04 smithi077 bash[80681]: audit 2024-09-13T00:51:04.292428+0000 mon.a (mon.0) 1391 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:51:04.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:04 smithi077 bash[83127]: cluster 2024-09-13T00:51:02.823973+0000 mgr.y (mgr.25304) 455 : cluster [DBG] pgmap v935: 105 pgs: 105 active+clean; 56 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 56 KiB/s rd, 9.1 MiB/s wr, 88 op/s 2024-09-13T00:51:04.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:04 smithi077 bash[83127]: audit 2024-09-13T00:51:03.558187+0000 mon.a (mon.0) 1385 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:04.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:04 smithi077 bash[83127]: audit 2024-09-13T00:51:03.565602+0000 mon.a (mon.0) 1386 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:04.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:04 smithi077 bash[83127]: audit 2024-09-13T00:51:03.733875+0000 mon.a (mon.0) 1387 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:04.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:04 smithi077 bash[83127]: audit 2024-09-13T00:51:03.844836+0000 mon.a (mon.0) 1388 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:04.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:04 smithi077 bash[83127]: audit 2024-09-13T00:51:03.851635+0000 mon.a (mon.0) 1389 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:04.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:04 smithi077 bash[83127]: audit 2024-09-13T00:51:04.290901+0000 mon.a (mon.0) 1390 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:51:04.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:04 smithi077 bash[83127]: audit 2024-09-13T00:51:04.292428+0000 mon.a (mon.0) 1391 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:51:04.956 INFO:tasks.workunit.client.0.smithi077.stdout: i=4 exists [11835209~1716033,16961709~4324172,25414253~303245,27215137~984633,33004547~1877145,44644623~7595311,62780271~112299,66532596~10917506,94755463~888673,104204950~2459588,120022934~8804768,132712348~6401017,167226701~1958761,177616439~6534770,207727920~4626596,214097157~6279146,230698020~664117,245374890~5017042,261630599~988124,266513173~7563069,275274515~10693812,288050963~12302193,312922432~4810361,321663886~5211040,333121499~2697631,343111568~2086045,350547027~2260147,363850683~866908,365523275~8337052,377438429~8928185,391179156~644734,393516028~4657160,401653630~3766966,408258776~4450348] wrote [16961709~4324172,69362580~8087522,94755463~888673,208670672~2384681,261630599~988124,266513173~7170792,288350280~12002876,333121499~2697631,391179156~644734] 2024-09-13T00:51:05.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:05 smithi088 bash[68960]: audit 2024-09-13T00:51:04.301970+0000 mon.a (mon.0) 1392 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:05.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:05 smithi088 bash[68960]: cluster 2024-09-13T00:51:04.858232+0000 mon.a (mon.0) 1393 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2024-09-13T00:51:05.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:05 smithi077 bash[80681]: audit 2024-09-13T00:51:04.301970+0000 mon.a (mon.0) 1392 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:05.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:05 smithi077 bash[80681]: cluster 2024-09-13T00:51:04.858232+0000 mon.a (mon.0) 1393 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2024-09-13T00:51:05.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:05 smithi077 bash[83127]: audit 2024-09-13T00:51:04.301970+0000 mon.a (mon.0) 1392 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:05.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:05 smithi077 bash[83127]: cluster 2024-09-13T00:51:04.858232+0000 mon.a (mon.0) 1393 : cluster [DBG] osdmap e862: 8 total, 8 up, 8 in 2024-09-13T00:51:05.947 INFO:tasks.workunit.client.0.smithi077.stdout: i=5 exists [11835209~1716033,16961709~4324172,25414253~303245,27215137~984633,30943519~5126877,44644623~7595311,62780271~112299,66532596~10917506,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~9084445,147675793~3299225,167226701~1958761,172643947~3012491,177616439~6534770,202003294~10351222,214097157~6279146,230698020~664117,239574985~792838,245374890~5017042,261630599~988124,266513173~7563069,275274515~10693812,288050963~12302193,312922432~4810361,321663886~5211040,333121499~2697631,343111568~2086045,350547027~2260147,363850683~866908,365523275~8337052,377438429~8928185,391179156~644734,393516028~4657160,401653630~3766966,406459211~6249913] wrote [30943519~5126877,111226631~4230026,124397052~1978614,130028920~4587913,147675793~3299225,172643947~3012491,202003294~7354786,239574985~792838,382830311~404953,406459211~2044500] 2024-09-13T00:51:06.176 INFO:tasks.workunit.client.0.smithi077.stdout: i=6 exists [11835209~1716033,16961709~4324172,25414253~303245,27215137~984633,30943519~5126877,44644623~7595311,62780271~112299,66532596~10917506,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~9084445,147675793~3299225,167226701~1958761,172643947~3012491,177616439~6534770,202003294~10351222,214097157~6279146,230698020~19693912,261630599~988124,266513173~7563069,275274515~10693812,288050963~12302193,312922432~7462051,321663886~5211040,333121499~2697631,337297297~1397679,343111568~2086045,350547027~2260147,363850683~866908,365523275~8337052,377438429~8928185,391179156~644734,393516028~4657160,401653630~3766966,406459211~6249913] wrote [46468688~2413894,67930859~945201,69006533~4513357,81411365~2058558,132271796~4762410,231065968~14680695,316777936~3606547,337297297~1397679] 2024-09-13T00:51:06.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:06 smithi088 bash[68960]: cluster 2024-09-13T00:51:04.825075+0000 mgr.y (mgr.25304) 456 : cluster [DBG] pgmap v937: 105 pgs: 105 active+clean; 209 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 107 KiB/s rd, 25 MiB/s wr, 179 op/s; 2.0 MiB/s, 1 objects/s recovering 2024-09-13T00:51:06.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:06 smithi088 bash[68960]: cluster 2024-09-13T00:51:05.859966+0000 mon.a (mon.0) 1394 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2024-09-13T00:51:06.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:06 smithi088 bash[68960]: cluster 2024-09-13T00:51:06.091577+0000 mon.a (mon.0) 1395 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2024-09-13T00:51:06.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:06 smithi077 bash[80681]: cluster 2024-09-13T00:51:04.825075+0000 mgr.y (mgr.25304) 456 : cluster [DBG] pgmap v937: 105 pgs: 105 active+clean; 209 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 107 KiB/s rd, 25 MiB/s wr, 179 op/s; 2.0 MiB/s, 1 objects/s recovering 2024-09-13T00:51:06.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:06 smithi077 bash[80681]: cluster 2024-09-13T00:51:05.859966+0000 mon.a (mon.0) 1394 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2024-09-13T00:51:06.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:06 smithi077 bash[80681]: cluster 2024-09-13T00:51:06.091577+0000 mon.a (mon.0) 1395 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2024-09-13T00:51:06.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:06 smithi077 bash[83127]: cluster 2024-09-13T00:51:04.825075+0000 mgr.y (mgr.25304) 456 : cluster [DBG] pgmap v937: 105 pgs: 105 active+clean; 209 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 107 KiB/s rd, 25 MiB/s wr, 179 op/s; 2.0 MiB/s, 1 objects/s recovering 2024-09-13T00:51:06.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:06 smithi077 bash[83127]: cluster 2024-09-13T00:51:05.859966+0000 mon.a (mon.0) 1394 : cluster [DBG] osdmap e863: 8 total, 8 up, 8 in 2024-09-13T00:51:06.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:06 smithi077 bash[83127]: cluster 2024-09-13T00:51:06.091577+0000 mon.a (mon.0) 1395 : cluster [DBG] osdmap e864: 8 total, 8 up, 8 in 2024-09-13T00:51:07.197 INFO:tasks.workunit.client.0.smithi077.stdout: i=7 exists [11835209~1716033,16961709~4324172,25414253~303245,27215137~984633,30943519~5126877,44644623~9706840,62780271~112299,66532596~10917506,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~9084445,147675793~3299225,167226701~1958761,172643947~11507262,192928176~5088291,202003294~10351222,214097157~6279146,230698020~19693912,256716141~6443241,266513173~7563069,275274515~10693812,288050963~15249772,312922432~13952494,333121499~2697631,337297297~1397679,343111568~2086045,350547027~2260147,363850683~866908,365523275~8337052,377438429~8928185,391179156~644734,393516028~4657160,401653630~3766966,406459211~6249913] wrote [49928767~4422696,136108718~1968740,175544318~2898368,192928176~5088291,237211987~11263825,256716141~6443241,297925649~5375086,320232816~2552469,368358100~1519512] 2024-09-13T00:51:08.235 INFO:tasks.workunit.client.0.smithi077.stdout: i=8 exists [11835209~1716033,16961709~8298723,25414253~303245,27215137~984633,30943519~5126877,44644623~9706840,62780271~112299,66532596~10917506,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~9084445,147675793~3299225,167226701~1958761,172643947~11507262,192928176~5088291,202003294~10351222,214097157~6279146,225502243~4133844,230698020~19693912,255387586~7771796,266513173~7563069,275274515~10693812,288050963~15249772,312922432~13952494,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,365523275~8337052,375685896~10680718,389497706~8675482,401653630~3766966,406459211~6249913] wrote [20041909~5218523,48987206~726746,225502243~4133844,255387586~7037457,295560575~7612635,335285595~1050423,350444844~7313146,375685896~3112271,389497706~7075119] 2024-09-13T00:51:08.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:08 smithi088 bash[68960]: cluster 2024-09-13T00:51:06.825906+0000 mgr.y (mgr.25304) 457 : cluster [DBG] pgmap v941: 105 pgs: 105 active+clean; 226 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 122 KiB/s rd, 27 MiB/s wr, 203 op/s; 2.3 MiB/s, 1 objects/s recovering 2024-09-13T00:51:08.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:08 smithi088 bash[68960]: cluster 2024-09-13T00:51:07.104680+0000 mon.a (mon.0) 1396 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2024-09-13T00:51:08.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:08 smithi077 bash[80681]: cluster 2024-09-13T00:51:06.825906+0000 mgr.y (mgr.25304) 457 : cluster [DBG] pgmap v941: 105 pgs: 105 active+clean; 226 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 122 KiB/s rd, 27 MiB/s wr, 203 op/s; 2.3 MiB/s, 1 objects/s recovering 2024-09-13T00:51:08.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:08 smithi077 bash[80681]: cluster 2024-09-13T00:51:07.104680+0000 mon.a (mon.0) 1396 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2024-09-13T00:51:08.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:08 smithi077 bash[83127]: cluster 2024-09-13T00:51:06.825906+0000 mgr.y (mgr.25304) 457 : cluster [DBG] pgmap v941: 105 pgs: 105 active+clean; 226 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 122 KiB/s rd, 27 MiB/s wr, 203 op/s; 2.3 MiB/s, 1 objects/s recovering 2024-09-13T00:51:08.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:08 smithi077 bash[83127]: cluster 2024-09-13T00:51:07.104680+0000 mon.a (mon.0) 1396 : cluster [DBG] osdmap e865: 8 total, 8 up, 8 in 2024-09-13T00:51:08.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:51:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:51:08] "GET /metrics HTTP/1.1" 200 36676 "" "Prometheus/2.43.0" 2024-09-13T00:51:09.246 INFO:tasks.workunit.client.0.smithi077.stdout: i=9 exists [7447179~4073964,11835209~1716033,16961709~8298723,25414253~303245,27215137~984633,30943519~5126877,41106664~13244799,62780271~112299,66532596~10917506,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~9084445,147138413~3836605,167226701~1958761,172643947~11507262,192928176~5088291,202003294~10351222,214097157~6279146,225502243~4133844,230698020~19693912,255387586~7771796,264304297~9771945,275274515~10693812,288050963~15249772,307269978~110347,308023339~18851587,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,365523275~8337052,375685896~10680718,389497706~8675482,401653630~3766966,406459211~6249913] wrote [7447179~4073964,41106664~5965111,147138413~922932,237941494~7012785,264304297~6430095,307269978~110347,308023339~7288248,377731437~2613201,390858265~5925901] 2024-09-13T00:51:09.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:09 smithi077 bash[80681]: cluster 2024-09-13T00:51:08.114999+0000 mon.a (mon.0) 1397 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2024-09-13T00:51:09.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:09 smithi077 bash[83127]: cluster 2024-09-13T00:51:08.114999+0000 mon.a (mon.0) 1397 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2024-09-13T00:51:09.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:09 smithi088 bash[68960]: cluster 2024-09-13T00:51:08.114999+0000 mon.a (mon.0) 1397 : cluster [DBG] osdmap e866: 8 total, 8 up, 8 in 2024-09-13T00:51:10.221 INFO:tasks.workunit.client.0.smithi077.stdout: i=10 exists [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,25414253~303245,27215137~984633,30943519~6589017,41106664~13244799,59919134~784701,62780271~112299,66532596~10917506,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~9084445,147138413~7483873,167226701~1958761,172643947~11507262,185145489~6850225,192928176~5088291,202003294~10351222,214097157~9213867,225502243~4133844,230698020~19693912,255387586~7771796,264304297~9771945,275274515~10693812,288050963~15249772,307269978~110347,308023339~19566249,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,365523275~8337052,375685896~10680718,389497706~8675482,401653630~3766966,406459211~6249913] wrote [1533919~394093,7753840~1657202,32783801~4748735,59919134~784701,147811050~6811236,185145489~6850225,218243585~5067439,249652551~224229,324279722~3309866] 2024-09-13T00:51:10.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:10 smithi077 bash[80681]: cluster 2024-09-13T00:51:08.826752+0000 mgr.y (mgr.25304) 458 : cluster [DBG] pgmap v944: 105 pgs: 105 active+clean; 276 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 135 KiB/s rd, 15 MiB/s wr, 210 op/s 2024-09-13T00:51:10.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:10 smithi077 bash[80681]: cluster 2024-09-13T00:51:09.125142+0000 mon.a (mon.0) 1398 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2024-09-13T00:51:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:10 smithi077 bash[83127]: cluster 2024-09-13T00:51:08.826752+0000 mgr.y (mgr.25304) 458 : cluster [DBG] pgmap v944: 105 pgs: 105 active+clean; 276 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 135 KiB/s rd, 15 MiB/s wr, 210 op/s 2024-09-13T00:51:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:10 smithi077 bash[83127]: cluster 2024-09-13T00:51:09.125142+0000 mon.a (mon.0) 1398 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2024-09-13T00:51:10.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:10 smithi088 bash[68960]: cluster 2024-09-13T00:51:08.826752+0000 mgr.y (mgr.25304) 458 : cluster [DBG] pgmap v944: 105 pgs: 105 active+clean; 276 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 135 KiB/s rd, 15 MiB/s wr, 210 op/s 2024-09-13T00:51:10.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:10 smithi088 bash[68960]: cluster 2024-09-13T00:51:09.125142+0000 mon.a (mon.0) 1398 : cluster [DBG] osdmap e867: 8 total, 8 up, 8 in 2024-09-13T00:51:11.248 INFO:tasks.workunit.client.0.smithi077.stdout: i=11 exists [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,25414253~303245,27215137~984633,29503754~8028782,41106664~13244799,59919134~784701,62780271~112299,64412830~13037272,78422427~5908409,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~9084445,147138413~7483873,167226701~16924508,185145489~6850225,192928176~8296411,202003294~10351222,214097157~9213867,225502243~4133844,230698020~20474268,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~19566249,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,365523275~8337052,375685896~10680718,389497706~8675482,401653630~3766966,406459211~6249913] wrote [20029799~2994873,29503754~5993847,64412830~6560139,78422427~5908409,169134757~5904995,195924737~5299850,245603211~5569077,273613045~1262583,335206066~179599] 2024-09-13T00:51:11.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:11 smithi077 bash[80681]: cluster 2024-09-13T00:51:10.133348+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2024-09-13T00:51:11.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:11 smithi077 bash[83127]: cluster 2024-09-13T00:51:10.133348+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2024-09-13T00:51:11.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:11 smithi088 bash[68960]: cluster 2024-09-13T00:51:10.133348+0000 mon.a (mon.0) 1399 : cluster [DBG] osdmap e868: 8 total, 8 up, 8 in 2024-09-13T00:51:12.293 INFO:tasks.workunit.client.0.smithi077.stdout: i=12 exists [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,25414253~303245,27215137~984633,29503754~8028782,41106664~13244799,59919134~784701,62780271~112299,64412830~13037272,78422427~11925226,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,129060064~10053301,147138413~7483873,167226701~16924508,185145489~6850225,192928176~8296411,202003294~10351222,214097157~9213867,225502243~4133844,230698020~20474268,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~4838512,343111568~2086045,348106239~9651751,363850683~866908,365523275~8337052,375685896~12489953,389497706~8675482,401653630~3766966,406071955~8147490] wrote [32229647~1744274,82234999~8112654,129060064~5449512,241666235~4926956,326698751~6075589,336808173~4838512,348106239~3988747,384302117~3873732,406071955~8147490] 2024-09-13T00:51:12.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:12 smithi077 bash[80681]: cluster 2024-09-13T00:51:10.827376+0000 mgr.y (mgr.25304) 459 : cluster [DBG] pgmap v947: 105 pgs: 105 active+clean; 443 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 49 MiB/s wr, 228 op/s 2024-09-13T00:51:12.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:12 smithi077 bash[80681]: cluster 2024-09-13T00:51:11.146747+0000 mon.a (mon.0) 1400 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2024-09-13T00:51:12.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:12 smithi077 bash[83127]: cluster 2024-09-13T00:51:10.827376+0000 mgr.y (mgr.25304) 459 : cluster [DBG] pgmap v947: 105 pgs: 105 active+clean; 443 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 49 MiB/s wr, 228 op/s 2024-09-13T00:51:12.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:12 smithi077 bash[83127]: cluster 2024-09-13T00:51:11.146747+0000 mon.a (mon.0) 1400 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2024-09-13T00:51:12.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:12 smithi088 bash[68960]: cluster 2024-09-13T00:51:10.827376+0000 mgr.y (mgr.25304) 459 : cluster [DBG] pgmap v947: 105 pgs: 105 active+clean; 443 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 136 KiB/s rd, 49 MiB/s wr, 228 op/s 2024-09-13T00:51:12.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:12 smithi088 bash[68960]: cluster 2024-09-13T00:51:11.146747+0000 mon.a (mon.0) 1400 : cluster [DBG] osdmap e869: 8 total, 8 up, 8 in 2024-09-13T00:51:12.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:51:12 smithi088 bash[61791]: ts=2024-09-13T00:51:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:51:13.292 INFO:tasks.workunit.client.0.smithi077.stdout: i=13 exists [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,25414253~303245,27215137~984633,29503754~8028782,41106664~17821373,59919134~784701,62780271~112299,64412830~13037272,78422427~11925226,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,129060064~10053301,147138413~7483873,167226701~16924508,185145489~6850225,192928176~8296411,202003294~10351222,214097157~9213867,225502243~4133844,230698020~20474268,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,365523275~8337052,375685896~22487292,401653630~3766966,406071955~8147490] wrote [54292152~4635885,89298043~942348,170779539~568367,176889496~3521015,264933973~599451,297168147~5073715,340517251~4565791,351050230~7478045,381207933~6058720,387676088~5905595] 2024-09-13T00:51:13.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:13 smithi077 bash[80681]: cluster 2024-09-13T00:51:12.146796+0000 mon.a (mon.0) 1401 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2024-09-13T00:51:13.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:13 smithi077 bash[83127]: cluster 2024-09-13T00:51:12.146796+0000 mon.a (mon.0) 1401 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2024-09-13T00:51:13.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:13 smithi088 bash[68960]: cluster 2024-09-13T00:51:12.146796+0000 mon.a (mon.0) 1401 : cluster [DBG] osdmap e870: 8 total, 8 up, 8 in 2024-09-13T00:51:14.317 INFO:tasks.workunit.client.0.smithi077.stdout: i=14 exists [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,25414253~303245,27215137~984633,29503754~8028782,41106664~17821373,59919134~784701,62780271~112299,64412830~13037272,78422427~12611916,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,120022934~19090431,147138413~7483873,167226701~16924508,185145489~16079098,202003294~10351222,214097157~15538930,230698020~20474268,253517969~879309,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,365523275~8337052,375685896~22487292,401653630~3766966,406071955~8147490] wrote [81835783~4369273,86388736~4645607,94079298~566015,126832157~6964267,171824257~4101766,189320558~5499828,221082790~7087134,253517969~879309,310505352~7387329,350426872~5051294] 2024-09-13T00:51:14.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:14 smithi077 bash[80681]: cluster 2024-09-13T00:51:12.828043+0000 mgr.y (mgr.25304) 460 : cluster [DBG] pgmap v950: 105 pgs: 105 active+clean; 462 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 40 MiB/s wr, 24 op/s 2024-09-13T00:51:14.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:14 smithi077 bash[80681]: cluster 2024-09-13T00:51:13.167246+0000 mon.a (mon.0) 1402 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2024-09-13T00:51:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:14 smithi077 bash[80681]: audit 2024-09-13T00:51:13.426293+0000 mon.a (mon.0) 1403 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:14 smithi077 bash[83127]: cluster 2024-09-13T00:51:12.828043+0000 mgr.y (mgr.25304) 460 : cluster [DBG] pgmap v950: 105 pgs: 105 active+clean; 462 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 40 MiB/s wr, 24 op/s 2024-09-13T00:51:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:14 smithi077 bash[83127]: cluster 2024-09-13T00:51:13.167246+0000 mon.a (mon.0) 1402 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2024-09-13T00:51:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:14 smithi077 bash[83127]: audit 2024-09-13T00:51:13.426293+0000 mon.a (mon.0) 1403 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:14 smithi088 bash[68960]: cluster 2024-09-13T00:51:12.828043+0000 mgr.y (mgr.25304) 460 : cluster [DBG] pgmap v950: 105 pgs: 105 active+clean; 462 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.5 KiB/s rd, 40 MiB/s wr, 24 op/s 2024-09-13T00:51:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:14 smithi088 bash[68960]: cluster 2024-09-13T00:51:13.167246+0000 mon.a (mon.0) 1402 : cluster [DBG] osdmap e871: 8 total, 8 up, 8 in 2024-09-13T00:51:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:14 smithi088 bash[68960]: audit 2024-09-13T00:51:13.426293+0000 mon.a (mon.0) 1403 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:15.297 INFO:tasks.workunit.client.0.smithi077.stdout: i=15 exists [1533919~9987224,11835209~1716033,16961709~8298723,25414253~303245,27215137~984633,29503754~8028782,41106664~17821373,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,120022934~19090431,143476416~11145870,167226701~16924508,185145489~16079098,201287152~11067364,214097157~15538930,230698020~20474268,253517969~879309,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,401653630~3766966,406071955~8147490] wrote [1657775~7078413,77484618~5025855,143476416~7974968,201287152~2371737,256862597~5111784,270407338~3576644,302702098~43268,326427340~4602689,399030489~1461454] 2024-09-13T00:51:15.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:15 smithi077 bash[80681]: cluster 2024-09-13T00:51:14.177263+0000 mon.a (mon.0) 1404 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2024-09-13T00:51:15.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:15 smithi077 bash[83127]: cluster 2024-09-13T00:51:14.177263+0000 mon.a (mon.0) 1404 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2024-09-13T00:51:15.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:15 smithi088 bash[68960]: cluster 2024-09-13T00:51:14.177263+0000 mon.a (mon.0) 1404 : cluster [DBG] osdmap e872: 8 total, 8 up, 8 in 2024-09-13T00:51:16.297 INFO:tasks.workunit.client.0.smithi077.stdout: i=16 exists [838088~10683055,11835209~1716033,16961709~8960196,27215137~984633,29503754~8028782,41106664~17821373,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,120022934~19090431,143476416~11145870,158413680~3974333,167226701~16924508,185145489~16079098,201287152~11067364,214097157~15538930,230698020~20474268,253517969~879309,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,401653630~3766966,406071955~8147490] wrote [838088~2598743,25074977~846928,133094772~5107867,158413680~3974333,185779876~5370913,227470207~930302,247799016~2251538,342946181~1901333,348003072~3151850] 2024-09-13T00:51:16.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:16 smithi077 bash[80681]: cluster 2024-09-13T00:51:14.829127+0000 mgr.y (mgr.25304) 461 : cluster [DBG] pgmap v953: 105 pgs: 105 active+clean; 648 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 49 MiB/s wr, 336 op/s 2024-09-13T00:51:16.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:16 smithi077 bash[80681]: cluster 2024-09-13T00:51:15.191047+0000 mon.a (mon.0) 1405 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2024-09-13T00:51:16.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:16 smithi077 bash[83127]: cluster 2024-09-13T00:51:14.829127+0000 mgr.y (mgr.25304) 461 : cluster [DBG] pgmap v953: 105 pgs: 105 active+clean; 648 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 49 MiB/s wr, 336 op/s 2024-09-13T00:51:16.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:16 smithi077 bash[83127]: cluster 2024-09-13T00:51:15.191047+0000 mon.a (mon.0) 1405 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2024-09-13T00:51:16.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:16 smithi088 bash[68960]: cluster 2024-09-13T00:51:14.829127+0000 mgr.y (mgr.25304) 461 : cluster [DBG] pgmap v953: 105 pgs: 105 active+clean; 648 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 49 MiB/s wr, 336 op/s 2024-09-13T00:51:16.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:16 smithi088 bash[68960]: cluster 2024-09-13T00:51:15.191047+0000 mon.a (mon.0) 1405 : cluster [DBG] osdmap e873: 8 total, 8 up, 8 in 2024-09-13T00:51:17.348 INFO:tasks.workunit.client.0.smithi077.stdout: i=17 exists [838088~10683055,11835209~1716033,16961709~8960196,27215137~984633,29503754~8028782,41106664~17821373,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,120022934~19090431,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~28348935,230698020~20584146,253517969~879309,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,401653630~3766966,406071955~8147490] wrote [151225375~7116299,185471528~3974527,203062023~14435328,244434952~6847214,264793518~2809034,276169467~2517149,349238425~6219209,379036633~2377445] 2024-09-13T00:51:17.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:17 smithi077 bash[80681]: cluster 2024-09-13T00:51:16.206418+0000 mon.a (mon.0) 1406 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2024-09-13T00:51:17.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:17 smithi077 bash[83127]: cluster 2024-09-13T00:51:16.206418+0000 mon.a (mon.0) 1406 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2024-09-13T00:51:17.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:17 smithi088 bash[68960]: cluster 2024-09-13T00:51:16.206418+0000 mon.a (mon.0) 1406 : cluster [DBG] osdmap e874: 8 total, 8 up, 8 in 2024-09-13T00:51:18.356 INFO:tasks.workunit.client.0.smithi077.stdout: i=18 exists [838088~10683055,11835209~1716033,16961709~8960196,27215137~984633,29503754~8028782,41106664~17821373,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~19090431,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~49995014,253517969~879309,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,401653630~3766966,406071955~8147490] wrote [27408852~677832,65799589~3651744,83698324~943169,104340251~6346965,112314922~3751122,221102177~11659672,265911692~5387971,278501790~7163536,387264267~95069] 2024-09-13T00:51:18.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:18 smithi077 bash[83127]: cluster 2024-09-13T00:51:16.829956+0000 mgr.y (mgr.25304) 462 : cluster [DBG] pgmap v956: 105 pgs: 105 active+clean; 658 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 49 MiB/s wr, 335 op/s 2024-09-13T00:51:18.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:18 smithi077 bash[83127]: cluster 2024-09-13T00:51:17.217925+0000 mon.a (mon.0) 1407 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2024-09-13T00:51:18.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:18 smithi077 bash[80681]: cluster 2024-09-13T00:51:16.829956+0000 mgr.y (mgr.25304) 462 : cluster [DBG] pgmap v956: 105 pgs: 105 active+clean; 658 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 49 MiB/s wr, 335 op/s 2024-09-13T00:51:18.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:18 smithi077 bash[80681]: cluster 2024-09-13T00:51:17.217925+0000 mon.a (mon.0) 1407 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2024-09-13T00:51:18.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:18 smithi088 bash[68960]: cluster 2024-09-13T00:51:16.829956+0000 mgr.y (mgr.25304) 462 : cluster [DBG] pgmap v956: 105 pgs: 105 active+clean; 658 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 209 KiB/s rd, 49 MiB/s wr, 335 op/s 2024-09-13T00:51:18.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:18 smithi088 bash[68960]: cluster 2024-09-13T00:51:17.217925+0000 mon.a (mon.0) 1407 : cluster [DBG] osdmap e875: 8 total, 8 up, 8 in 2024-09-13T00:51:18.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:51:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:51:18] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.43.0" 2024-09-13T00:51:19.343 INFO:tasks.workunit.client.0.smithi077.stdout: i=19 exists [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~17821373,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~19090431,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~49995014,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,401653630~3766966,406071955~8147490] wrote [15367686~2169275,42505574~5876247,69896487~3475498,147709657~1438140,222453969~178143,227622941~5962306,283145518~3513175,322349784~3676554,347057573~11931083] 2024-09-13T00:51:19.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:19 smithi077 bash[80681]: cluster 2024-09-13T00:51:18.220645+0000 mon.a (mon.0) 1408 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2024-09-13T00:51:19.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:19 smithi077 bash[83127]: cluster 2024-09-13T00:51:18.220645+0000 mon.a (mon.0) 1408 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2024-09-13T00:51:19.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:19 smithi088 bash[68960]: cluster 2024-09-13T00:51:18.220645+0000 mon.a (mon.0) 1408 : cluster [DBG] osdmap e876: 8 total, 8 up, 8 in 2024-09-13T00:51:20.265 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 1 diff [120022934~6051327,207727920~4626596,214186959~425193,277817514~8150813,350547027~2260147,365523275~6858519,393516028~4657160,408258776~4450348] 2024-09-13T00:51:20.266 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [120022934~6051327,207727920~4626596,214186959~425193,277817514~8150813,350547027~2260147,365523275~6858519,393516028~4657160,408258776~4450348] 2024-09-13T00:51:20.316 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [117440512~8633749,205520896~6833620,214186959~425193,276824064~9144263,348127232~4679942,364904448~7477346,390070272~8102916,406847488~5861636] 2024-09-13T00:51:20.316 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 2 diff [11835209~1716033,27215137~984633,33004547~1877145,62780271~112299,120022934~8804768,167226701~1958761,177616439~6534770,207727920~4626596,214186959~425193,275875051~10093276,288050963~868686,312922432~4810361,350547027~2260147,365523275~6858519,393516028~4657160,408258776~4450348] 2024-09-13T00:51:20.317 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [11835209~1716033,27215137~984633,33004547~1877145,62780271~112299,120022934~8804768,167226701~1958761,177616439~6534770,207727920~4626596,214186959~425193,275875051~10093276,288050963~868686,312922432~4810361,350547027~2260147,365523275~6858519,393516028~4657160,408258776~4450348] 2024-09-13T00:51:20.333 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [8388608~5162634,25717498~2482272,29360128~5521564,58720256~4172314,117440512~11387190,163577856~5607606,176160768~7990441,205520896~6833620,214186959~425193,272629760~16289889,310378496~7354297,348127232~4679942,364904448~7477346,390070272~8102916,406847488~5861636] 2024-09-13T00:51:20.333 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 3 diff [11835209~1716033,27215137~984633,33004547~1877145,62780271~112299,104204950~2459588,120022934~8804768,167226701~1958761,177616439~6534770,207727920~4626596,214186959~425193,268623601~2612185,272690745~1385497,275274515~10693812,288050963~868686,312922432~4810361,321663886~5211040,343111568~2086045,350547027~2260147,363850683~866908,365523275~8337052,385267053~1099561,393516028~4657160,408258776~4450348] 2024-09-13T00:51:20.333 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [11835209~1716033,27215137~984633,33004547~1877145,62780271~112299,104204950~2459588,120022934~8804768,167226701~1958761,177616439~6534770,207727920~4626596,214186959~425193,268623601~2612185,272690745~1385497,275274515~10693812,288050963~868686,312922432~4810361,321663886~5211040,343111568~2086045,350547027~2260147,363850683~866908,365523275~8337052,385267053~1099561,393516028~4657160,408258776~4450348] 2024-09-13T00:51:20.349 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [8388608~5162634,25717498~2482272,29360128~5521564,58720256~4172314,100663296~6001242,117440512~11387190,163577856~5607606,176160768~7990441,205520896~6833620,214186959~425193,268435456~2800330,272629760~16289889,310378496~7354297,318767104~8107822,339738624~5458989,348127232~4679942,360710144~4007447,364904448~8955879,385267053~1099561,390070272~8102916,406847488~5861636] 2024-09-13T00:51:20.349 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 4 diff [11835209~1716033,16961709~4324172,27215137~984633,33004547~1877145,62780271~112299,69362580~8087522,94755463~888673,104204950~2459588,120022934~8804768,167226701~1958761,177616439~6534770,207727920~4626596,214186959~425193,261630599~988124,266513173~7563069,275274515~10693812,288050963~12302193,312922432~4810361,321663886~5211040,333121499~2697631,343111568~2086045,350547027~2260147,363850683~866908,365523275~8337052,385267053~1099561,391179156~644734,393516028~4657160,408258776~4450348] 2024-09-13T00:51:20.349 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [11835209~1716033,16961709~4324172,27215137~984633,33004547~1877145,62780271~112299,69362580~8087522,94755463~888673,104204950~2459588,120022934~8804768,167226701~1958761,177616439~6534770,207727920~4626596,214186959~425193,261630599~988124,266513173~7563069,275274515~10693812,288050963~12302193,312922432~4810361,321663886~5211040,333121499~2697631,343111568~2086045,350547027~2260147,363850683~866908,365523275~8337052,385267053~1099561,391179156~644734,393516028~4657160,408258776~4450348] 2024-09-13T00:51:20.362 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [8388608~5162634,16777216~4508665,25717498~2482272,29360128~5521564,58720256~4172314,69362580~8087522,92274688~3369448,100663296~6001242,117440512~11387190,163577856~5607606,176160768~7990441,205520896~6833620,214186959~425193,260046848~2571875,264241152~36112004,310378496~7354297,318767104~8107822,331350016~4469114,339738624~5458989,348127232~4679942,360710144~4007447,364904448~8955879,385267053~1099561,390070272~8102916,406847488~5861636] 2024-09-13T00:51:20.363 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 5 diff [11835209~1716033,16961709~4324172,27215137~984633,30943519~5126877,62780271~112299,69362580~8087522,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~4587913,147675793~3299225,167226701~1958761,172643947~3012491,177616439~6534770,202003294~10351222,214186959~425193,239574985~792838,261630599~988124,266513173~7563069,275274515~10693812,288050963~12302193,312922432~4810361,321663886~5211040,333121499~2697631,343111568~2086045,350547027~2260147,363850683~866908,365523275~8337052,382830311~404953,385267053~1099561,391179156~644734,393516028~4657160,406459211~6249913] 2024-09-13T00:51:20.363 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [11835209~1716033,16961709~4324172,27215137~984633,30943519~5126877,62780271~112299,69362580~8087522,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~4587913,147675793~3299225,167226701~1958761,172643947~3012491,177616439~6534770,202003294~10351222,214186959~425193,239574985~792838,261630599~988124,266513173~7563069,275274515~10693812,288050963~12302193,312922432~4810361,321663886~5211040,333121499~2697631,343111568~2086045,350547027~2260147,363850683~866908,365523275~8337052,382830311~404953,385267053~1099561,391179156~644734,393516028~4657160,406459211~6249913] 2024-09-13T00:51:20.375 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [8388608~5162634,16777216~4508665,25717498~2482272,29360128~6710268,58720256~4172314,69362580~8087522,92274688~3369448,100663296~6001242,109051904~6404753,117440512~11387190,130028920~4587913,146800640~4174378,163577856~5607606,171966464~3689974,176160768~7990441,201326592~11027924,214186959~425193,239075328~1292495,260046848~2571875,264241152~36112004,310378496~7354297,318767104~8107822,331350016~4469114,339738624~5458989,348127232~4679942,360710144~4007447,364904448~8955879,382830311~404953,385267053~1099561,390070272~8102916,405420596~7288528] 2024-09-13T00:51:20.375 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 6 diff [11835209~1716033,16961709~4324172,27215137~984633,30943519~5126877,46468688~2413894,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~7005286,147675793~3299225,167226701~1958761,172643947~3012491,177616439~6534770,202003294~10351222,214186959~425193,231065968~14680695,261630599~988124,266513173~7563069,275274515~10693812,288050963~12302193,312922432~7462051,321663886~5211040,333121499~2697631,337297297~1397679,343111568~2086045,350547027~2260147,363850683~866908,365523275~8337052,382830311~404953,385267053~1099561,391179156~644734,393516028~4657160,406459211~6249913] 2024-09-13T00:51:20.375 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [11835209~1716033,16961709~4324172,27215137~984633,30943519~5126877,46468688~2413894,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~7005286,147675793~3299225,167226701~1958761,172643947~3012491,177616439~6534770,202003294~10351222,214186959~425193,231065968~14680695,261630599~988124,266513173~7563069,275274515~10693812,288050963~12302193,312922432~7462051,321663886~5211040,333121499~2697631,337297297~1397679,343111568~2086045,350547027~2260147,363850683~866908,365523275~8337052,382830311~404953,385267053~1099561,391179156~644734,393516028~4657160,406459211~6249913] 2024-09-13T00:51:20.387 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [8388608~5162634,16777216~4508665,25717498~2482272,29360128~6710268,46468688~2413894,58720256~4172314,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,100663296~6001242,109051904~6404753,117440512~11387190,130028920~7005286,146800640~4174378,163577856~5607606,171966464~3689974,176160768~7990441,201326592~11027924,214186959~425193,231065968~14680695,260046848~2571875,264241152~36112004,310378496~16496430,331350016~7344960,339738624~5458989,348127232~4679942,360710144~4007447,364904448~8955879,382830311~404953,385267053~1099561,390070272~8102916,405420596~7288528] 2024-09-13T00:51:20.387 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 7 diff [11835209~1716033,16961709~4324172,27215137~984633,30943519~5126877,46468688~2413894,49928767~4422696,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~8048538,147675793~3299225,167226701~1958761,172643947~11507262,192928176~5088291,202003294~10351222,214186959~425193,231065968~17409844,256716141~6443241,266513173~7563069,275274515~10693812,288050963~15249772,312922432~13952494,333121499~2697631,337297297~1397679,343111568~2086045,350547027~2260147,363850683~866908,365523275~8337052,382830311~404953,385267053~1099561,391179156~644734,393516028~4657160,406459211~6249913] 2024-09-13T00:51:20.387 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [11835209~1716033,16961709~4324172,27215137~984633,30943519~5126877,46468688~2413894,49928767~4422696,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~8048538,147675793~3299225,167226701~1958761,172643947~11507262,192928176~5088291,202003294~10351222,214186959~425193,231065968~17409844,256716141~6443241,266513173~7563069,275274515~10693812,288050963~15249772,312922432~13952494,333121499~2697631,337297297~1397679,343111568~2086045,350547027~2260147,363850683~866908,365523275~8337052,382830311~404953,385267053~1099561,391179156~644734,393516028~4657160,406459211~6249913] 2024-09-13T00:51:20.398 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [8388608~5162634,16777216~4508665,25717498~2482272,29360128~6710268,46468688~2413894,49928767~4422696,58720256~4172314,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,100663296~6001242,109051904~6404753,117440512~11387190,130028920~8048538,146800640~4174378,163577856~5607606,171966464~12184745,188743680~9272787,201326592~11027924,214186959~425193,231065968~17409844,255852544~7306838,264241152~39059583,310378496~16496430,331350016~7344960,339738624~5458989,348127232~4679942,360710144~4007447,364904448~8955879,382830311~404953,385267053~1099561,390070272~8102916,405420596~7288528] 2024-09-13T00:51:20.398 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 8 diff [11835209~1716033,16961709~8298723,27215137~984633,30943519~5126877,46468688~2413894,48987206~726746,49928767~4422696,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~8048538,147675793~3299225,167226701~1958761,172643947~11507262,192928176~5088291,202003294~10351222,214186959~425193,225502243~4133844,231065968~17409844,255387586~7771796,266513173~7563069,275274515~10693812,288050963~15249772,312922432~13952494,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,365523275~8337052,375685896~3112271,382830311~404953,385267053~1099561,389497706~8675482,406459211~6249913] 2024-09-13T00:51:20.399 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [11835209~1716033,16961709~8298723,27215137~984633,30943519~5126877,46468688~2413894,48987206~726746,49928767~4422696,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~8048538,147675793~3299225,167226701~1958761,172643947~11507262,192928176~5088291,202003294~10351222,214186959~425193,225502243~4133844,231065968~17409844,255387586~7771796,266513173~7563069,275274515~10693812,288050963~15249772,312922432~13952494,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,365523275~8337052,375685896~3112271,382830311~404953,385267053~1099561,389497706~8675482,406459211~6249913] 2024-09-13T00:51:20.409 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [8388608~5162634,16777216~8483216,25717498~2482272,29360128~6710268,46468688~2413894,48987206~726746,49928767~4422696,58720256~4172314,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,100663296~6001242,109051904~6404753,117440512~11387190,130028920~8048538,146800640~4174378,163577856~5607606,171966464~12184745,188743680~9272787,201326592~11027924,214186959~425193,222298112~7337975,231065968~17409844,251658240~11501142,264241152~39059583,310378496~16496430,331350016~7344960,339738624~5458989,348127232~9630758,360710144~4007447,364904448~8955879,375685896~3112271,382830311~404953,385267053~12906135,405420596~7288528] 2024-09-13T00:51:20.410 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 9 diff [7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,30943519~5126877,41106664~7775918,48987206~726746,49928767~4422696,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~8048538,147138413~3836605,167226701~1958761,172643947~11507262,192928176~5088291,202003294~10351222,214186959~425193,225502243~4133844,231065968~17409844,255387586~7771796,264304297~9771945,275274515~10693812,288050963~15249772,307269978~110347,308023339~18851587,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,365523275~8337052,375685896~4658742,382830311~404953,385267053~1099561,389497706~8675482,406459211~6249913] 2024-09-13T00:51:20.410 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,30943519~5126877,41106664~7775918,48987206~726746,49928767~4422696,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~8048538,147138413~3836605,167226701~1958761,172643947~11507262,192928176~5088291,202003294~10351222,214186959~425193,225502243~4133844,231065968~17409844,255387586~7771796,264304297~9771945,275274515~10693812,288050963~15249772,307269978~110347,308023339~18851587,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,365523275~8337052,375685896~4658742,382830311~404953,385267053~1099561,389497706~8675482,406459211~6249913] 2024-09-13T00:51:20.419 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~9356938,16777216~8483216,25717498~2482272,29360128~6710268,37748736~11133846,48987206~726746,49928767~4422696,58720256~4172314,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,100663296~6001242,109051904~6404753,117440512~11387190,130028920~8048538,146800640~4174378,163577856~5607606,171966464~12184745,188743680~9272787,201326592~11027924,214186959~425193,222298112~7337975,231065968~17409844,251658240~11501142,264241152~39059583,306184192~20690734,331350016~7344960,339738624~5458989,348127232~9630758,360710144~4007447,364904448~8955879,375685896~4658742,382830311~404953,385267053~12906135,405420596~7288528] 2024-09-13T00:51:20.419 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 10 diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,30943519~6589017,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~8048538,147138413~7483873,167226701~1958761,172643947~11507262,185145489~6850225,192928176~5088291,202003294~10351222,214186959~425193,218243585~5067439,225502243~4133844,231065968~17409844,249652551~224229,255387586~7771796,264304297~9771945,275274515~10693812,288050963~15249772,307269978~110347,308023339~19566249,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,365523275~8337052,375685896~4658742,382830311~404953,385267053~1099561,389497706~8675482,406459211~6249913] 2024-09-13T00:51:20.419 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,30943519~6589017,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~8048538,147138413~7483873,167226701~1958761,172643947~11507262,185145489~6850225,192928176~5088291,202003294~10351222,214186959~425193,218243585~5067439,225502243~4133844,231065968~17409844,249652551~224229,255387586~7771796,264304297~9771945,275274515~10693812,288050963~15249772,307269978~110347,308023339~19566249,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,365523275~8337052,375685896~4658742,382830311~404953,385267053~1099561,389497706~8675482,406459211~6249913] 2024-09-13T00:51:20.428 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~9356938,16777216~8483216,25717498~2482272,29360128~8172408,37748736~11133846,48987206~726746,49928767~4422696,58720256~4172314,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,100663296~6001242,109051904~6404753,117440512~11387190,130028920~8048538,146800640~7821646,163577856~5607606,171966464~12184745,184549376~13467091,201326592~11027924,214186959~425193,218243585~11392502,231065968~17409844,249652551~224229,251658240~11501142,264241152~39059583,306184192~21405396,331350016~7344960,339738624~5458989,348127232~9630758,360710144~4007447,364904448~8955879,375685896~4658742,382830311~404953,385267053~12906135,405420596~7288528] 2024-09-13T00:51:20.428 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 11 diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,62780271~112299,64412830~13037272,78422427~5908409,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~8048538,147138413~7483873,167226701~16924508,185145489~6850225,192928176~8296411,202003294~10351222,214186959~425193,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~19566249,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,365523275~8337052,375685896~4658742,382830311~404953,385267053~1099561,389497706~8675482,406459211~6249913] 2024-09-13T00:51:20.429 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,62780271~112299,64412830~13037272,78422427~5908409,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,130028920~8048538,147138413~7483873,167226701~16924508,185145489~6850225,192928176~8296411,202003294~10351222,214186959~425193,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~19566249,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,365523275~8337052,375685896~4658742,382830311~404953,385267053~1099561,389497706~8675482,406459211~6249913] 2024-09-13T00:51:20.437 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~9356938,16777216~8483216,25717498~2482272,29360128~8172408,37748736~11133846,48987206~726746,49928767~4422696,58720256~4172314,64412830~19918006,92274688~3369448,100663296~6001242,109051904~6404753,117440512~11387190,130028920~8048538,146800640~7821646,163577856~20573353,184549376~16675211,201326592~11027924,214186959~425193,218243585~11392502,231065968~20106320,251658240~11501142,264241152~39059583,306184192~21405396,331350016~7344960,339738624~5458989,348127232~9630758,360710144~4007447,364904448~8955879,375685896~4658742,382830311~404953,385267053~12906135,405420596~7288528] 2024-09-13T00:51:20.437 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 12 diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,62780271~112299,64412830~13037272,78422427~11925226,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,129060064~9017394,147138413~7483873,167226701~16924508,185145489~6850225,192928176~8296411,202003294~10351222,214186959~425193,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~4838512,343111568~2086045,348106239~9651751,363850683~866908,365523275~8337052,375685896~4658742,382830311~404953,384302117~3873732,389497706~8675482,406071955~8147490] 2024-09-13T00:51:20.437 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,62780271~112299,64412830~13037272,78422427~11925226,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,129060064~9017394,147138413~7483873,167226701~16924508,185145489~6850225,192928176~8296411,202003294~10351222,214186959~425193,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~4838512,343111568~2086045,348106239~9651751,363850683~866908,365523275~8337052,375685896~4658742,382830311~404953,384302117~3873732,389497706~8675482,406071955~8147490] 2024-09-13T00:51:20.445 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~9356938,16777216~8483216,25717498~2482272,29360128~8172408,37748736~11133846,48987206~726746,49928767~4422696,58720256~4172314,64412830~25934823,92274688~3369448,100663296~6001242,109051904~6404753,117440512~20636946,146800640~7821646,163577856~20573353,184549376~16675211,201326592~11027924,214186959~425193,218243585~11392502,231065968~20106320,251658240~11501142,264241152~39059583,306184192~51573798,360710144~4007447,364904448~8955879,375685896~4658742,382830311~404953,384302117~13871071,405420596~8798849] 2024-09-13T00:51:20.445 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 13 diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,78422427~11925226,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,129060064~9017394,147138413~7483873,167226701~16924508,185145489~6850225,192928176~8296411,202003294~10351222,214186959~425193,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,365523275~8337052,375685896~4658742,381207933~16965255,406071955~8147490] 2024-09-13T00:51:20.445 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,78422427~11925226,94755463~888673,104204950~2459588,111226631~4230026,120022934~8804768,129060064~9017394,147138413~7483873,167226701~16924508,185145489~6850225,192928176~8296411,202003294~10351222,214186959~425193,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,365523275~8337052,375685896~4658742,381207933~16965255,406071955~8147490] 2024-09-13T00:51:20.454 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~9356938,16777216~8483216,25717498~2482272,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~25934823,92274688~3369448,100663296~6001242,109051904~6404753,117440512~20636946,146800640~7821646,163577856~20573353,184549376~16675211,201326592~11027924,214186959~425193,218243585~11392502,231065968~20106320,251658240~11501142,264241152~39059583,306184192~52344083,360710144~4007447,364904448~8955879,375685896~4658742,381207933~16965255,405420596~8798849] 2024-09-13T00:51:20.454 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 14 diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,78422427~12611916,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,120022934~18054524,147138413~7483873,167226701~16924508,185145489~16079098,202003294~10351222,214186959~425193,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,365523275~8337052,375685896~4658742,381207933~16965255,406071955~8147490] 2024-09-13T00:51:20.454 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,78422427~12611916,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,120022934~18054524,147138413~7483873,167226701~16924508,185145489~16079098,202003294~10351222,214186959~425193,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,365523275~8337052,375685896~4658742,381207933~16965255,406071955~8147490] 2024-09-13T00:51:20.462 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~9356938,16777216~8483216,25717498~2482272,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~6001242,109051904~6404753,117440512~20636946,146800640~7821646,163577856~20573353,184549376~16675211,201326592~11027924,214186959~425193,218243585~11392502,231065968~20106320,251658240~11501142,264241152~39059583,306184192~52344083,360710144~4007447,364904448~8955879,375685896~4658742,381207933~16965255,405420596~8798849] 2024-09-13T00:51:20.462 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 15 diff [1533919~9987224,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,120022934~18054524,143476416~11145870,167226701~16924508,185145489~16079098,201287152~11067364,214186959~425193,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,365523275~8337052,375685896~4658742,381207933~16965255,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.462 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~9987224,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,120022934~18054524,143476416~11145870,167226701~16924508,185145489~16079098,201287152~11067364,214186959~425193,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,365523275~8337052,375685896~4658742,381207933~16965255,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.470 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~13551242,16777216~8483216,25717498~2482272,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~6001242,109051904~6404753,117440512~20636946,142606336~12015950,163577856~20573353,184549376~27805140,214186959~425193,218243585~11392502,231065968~20106320,251658240~11501142,264241152~39059583,306184192~52344083,360710144~4007447,364904448~8955879,375685896~4658742,381207933~16965255,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.470 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 16 diff [838088~10683055,11835209~1716033,16961709~8960196,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,120022934~18179705,143476416~11145870,158413680~3974333,167226701~16924508,185145489~16079098,201287152~11067364,214186959~425193,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,365523275~8337052,375685896~4658742,381207933~16965255,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.470 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,16961709~8960196,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,120022934~18179705,143476416~11145870,158413680~3974333,167226701~16924508,185145489~16079098,201287152~11067364,214186959~425193,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,365523275~8337052,375685896~4658742,381207933~16965255,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.478 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~13551242,16777216~11422554,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~6001242,109051904~6404753,117440512~20762127,142606336~12015950,155189248~7198765,163577856~20573353,184549376~27805140,214186959~425193,218243585~11392502,231065968~20106320,251658240~11501142,264241152~39059583,306184192~52344083,360710144~4007447,364904448~8955879,375685896~4658742,381207933~16965255,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.478 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 17 diff [838088~10683055,11835209~1716033,16961709~8960196,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~11392502,231065968~20216198,253517969~879309,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.479 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,16961709~8960196,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~11392502,231065968~20216198,253517969~879309,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.486 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~13551242,16777216~11422554,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~6001242,109051904~6404753,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~11392502,231065968~20216198,251658240~11501142,264241152~39059583,306184192~52344083,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.486 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 18 diff [838088~10683055,11835209~1716033,16961709~8960196,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.486 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,16961709~8960196,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~10693812,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.494 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~13551242,16777216~11422554,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52344083,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.494 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 19 diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.494 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.501 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.502 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 2 diff [11835209~1716033,27215137~984633,33004547~1877145,62780271~112299,121899613~6928089,167226701~1958761,177616439~6534770,275875051~7619440,288050963~868686,312922432~4810361] 2024-09-13T00:51:20.502 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [11835209~1716033,27215137~984633,33004547~1877145,62780271~112299,121899613~6928089,167226701~1958761,177616439~6534770,275875051~7619440,288050963~868686,312922432~4810361] 2024-09-13T00:51:20.510 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [8388608~5162634,25717498~2482272,29360128~5521564,58720256~4172314,121899613~6928089,163577856~5607606,176160768~7990441,272629760~10864731,285968327~2951322,310378496~7354297] 2024-09-13T00:51:20.510 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 3 diff [11835209~1716033,27215137~984633,33004547~1877145,62780271~112299,104204950~2459588,121899613~6928089,167226701~1958761,177616439~6534770,268623601~2612185,272690745~1385497,275274515~8219976,288050963~868686,312922432~4810361,321663886~5211040,343111568~2086045,363850683~866908,366062574~7797753,385267053~1099561,409582788~931046] 2024-09-13T00:51:20.510 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [11835209~1716033,27215137~984633,33004547~1877145,62780271~112299,104204950~2459588,121899613~6928089,167226701~1958761,177616439~6534770,268623601~2612185,272690745~1385497,275274515~8219976,288050963~868686,312922432~4810361,321663886~5211040,343111568~2086045,363850683~866908,366062574~7797753,385267053~1099561,409582788~931046] 2024-09-13T00:51:20.517 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [8388608~5162634,25717498~2482272,29360128~5521564,58720256~4172314,100663296~6001242,121899613~6928089,163577856~5607606,176160768~7990441,268435456~2800330,272629760~10864731,285968327~2951322,310378496~7354297,318767104~8107822,339738624~5458989,360710144~4007447,366062574~7797753,385267053~1099561,409582788~931046] 2024-09-13T00:51:20.517 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 4 diff [11835209~1716033,16961709~4324172,27215137~984633,33004547~1877145,62780271~112299,69362580~8087522,94755463~888673,104204950~2459588,121899613~6928089,167226701~1958761,177616439~6534770,208670672~2384681,261630599~988124,266513173~7563069,275274515~8219976,288050963~12302193,312922432~4810361,321663886~5211040,333121499~2697631,343111568~2086045,363850683~866908,366062574~7797753,385267053~1099561,391179156~644734,409582788~931046] 2024-09-13T00:51:20.517 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [11835209~1716033,16961709~4324172,27215137~984633,33004547~1877145,62780271~112299,69362580~8087522,94755463~888673,104204950~2459588,121899613~6928089,167226701~1958761,177616439~6534770,208670672~2384681,261630599~988124,266513173~7563069,275274515~8219976,288050963~12302193,312922432~4810361,321663886~5211040,333121499~2697631,343111568~2086045,363850683~866908,366062574~7797753,385267053~1099561,391179156~644734,409582788~931046] 2024-09-13T00:51:20.524 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [8388608~5162634,16777216~4508665,25717498~2482272,29360128~5521564,58720256~4172314,69362580~8087522,92274688~3369448,100663296~6001242,121899613~6928089,163577856~5607606,176160768~7990441,208670672~2384681,260046848~2571875,264241152~19253339,285968327~14384829,310378496~7354297,318767104~8107822,331350016~4469114,339738624~5458989,360710144~4007447,366062574~7797753,385267053~1099561,391179156~644734,409582788~931046] 2024-09-13T00:51:20.524 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 5 diff [11835209~1716033,16961709~4324172,27215137~984633,30943519~5126877,62780271~112299,69362580~8087522,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,130028920~4587913,147675793~3299225,167226701~1958761,172643947~3012491,177616439~6534770,202003294~9052059,239574985~792838,261630599~988124,266513173~7563069,275274515~8219976,288050963~12302193,312922432~4810361,321663886~5211040,333121499~2697631,343111568~2086045,363850683~866908,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.524 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [11835209~1716033,16961709~4324172,27215137~984633,30943519~5126877,62780271~112299,69362580~8087522,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,130028920~4587913,147675793~3299225,167226701~1958761,172643947~3012491,177616439~6534770,202003294~9052059,239574985~792838,261630599~988124,266513173~7563069,275274515~8219976,288050963~12302193,312922432~4810361,321663886~5211040,333121499~2697631,343111568~2086045,363850683~866908,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:20 smithi077 bash[80681]: cluster 2024-09-13T00:51:18.830799+0000 mgr.y (mgr.25304) 463 : cluster [DBG] pgmap v959: 105 pgs: 105 active+clean; 710 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 232 KiB/s rd, 17 MiB/s wr, 347 op/s 2024-09-13T00:51:20.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:20 smithi077 bash[80681]: cluster 2024-09-13T00:51:19.241425+0000 mon.a (mon.0) 1409 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2024-09-13T00:51:20.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:20 smithi077 bash[83127]: cluster 2024-09-13T00:51:18.830799+0000 mgr.y (mgr.25304) 463 : cluster [DBG] pgmap v959: 105 pgs: 105 active+clean; 710 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 232 KiB/s rd, 17 MiB/s wr, 347 op/s 2024-09-13T00:51:20.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:20 smithi077 bash[83127]: cluster 2024-09-13T00:51:19.241425+0000 mon.a (mon.0) 1409 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2024-09-13T00:51:20.532 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [8388608~5162634,16777216~4508665,25717498~2482272,29360128~6710268,58720256~4172314,69362580~8087522,92274688~3369448,100663296~6001242,109051904~6404753,121899613~6928089,130028920~4587913,146800640~4174378,163577856~5607606,171966464~3689974,176160768~7990441,201326592~9728761,239075328~1292495,260046848~2571875,264241152~19253339,285968327~14384829,310378496~7354297,318767104~8107822,331350016~4469114,339738624~5458989,360710144~4007447,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,405420596~3083115,409582788~931046] 2024-09-13T00:51:20.532 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 6 diff [11835209~1716033,16961709~4324172,27215137~984633,30943519~5126877,46468688~2413894,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,130028920~7005286,147675793~3299225,167226701~1958761,172643947~3012491,177616439~6534770,202003294~9052059,231065968~14680695,261630599~988124,266513173~7563069,275274515~8219976,288050963~12302193,312922432~7462051,321663886~5211040,333121499~2697631,337297297~1397679,343111568~2086045,363850683~866908,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.532 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [11835209~1716033,16961709~4324172,27215137~984633,30943519~5126877,46468688~2413894,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,130028920~7005286,147675793~3299225,167226701~1958761,172643947~3012491,177616439~6534770,202003294~9052059,231065968~14680695,261630599~988124,266513173~7563069,275274515~8219976,288050963~12302193,312922432~7462051,321663886~5211040,333121499~2697631,337297297~1397679,343111568~2086045,363850683~866908,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.539 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [8388608~5162634,16777216~4508665,25717498~2482272,29360128~6710268,46468688~2413894,58720256~4172314,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,100663296~6001242,109051904~6404753,121899613~6928089,130028920~7005286,146800640~4174378,163577856~5607606,171966464~3689974,176160768~7990441,201326592~9728761,231065968~14680695,260046848~2571875,264241152~19253339,285968327~14384829,310378496~16496430,331350016~7344960,339738624~5458989,360710144~4007447,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,405420596~3083115,409582788~931046] 2024-09-13T00:51:20.539 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 7 diff [11835209~1716033,16961709~4324172,27215137~984633,30943519~5126877,46468688~2413894,49928767~4422696,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,130028920~8048538,147675793~3299225,167226701~1958761,172643947~11507262,192928176~5088291,202003294~9052059,231065968~17409844,256716141~6443241,266513173~7563069,275274515~8219976,288050963~15249772,312922432~13952494,333121499~2697631,337297297~1397679,343111568~2086045,363850683~866908,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.539 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [11835209~1716033,16961709~4324172,27215137~984633,30943519~5126877,46468688~2413894,49928767~4422696,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,130028920~8048538,147675793~3299225,167226701~1958761,172643947~11507262,192928176~5088291,202003294~9052059,231065968~17409844,256716141~6443241,266513173~7563069,275274515~8219976,288050963~15249772,312922432~13952494,333121499~2697631,337297297~1397679,343111568~2086045,363850683~866908,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.546 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [8388608~5162634,16777216~4508665,25717498~2482272,29360128~6710268,46468688~2413894,49928767~4422696,58720256~4172314,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,100663296~6001242,109051904~6404753,121899613~6928089,130028920~8048538,146800640~4174378,163577856~5607606,171966464~12184745,188743680~9272787,201326592~9728761,231065968~17409844,255852544~7306838,264241152~19253339,285968327~17332408,310378496~16496430,331350016~7344960,339738624~5458989,360710144~4007447,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,405420596~3083115,409582788~931046] 2024-09-13T00:51:20.546 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 8 diff [11835209~1716033,16961709~8298723,27215137~984633,30943519~5126877,46468688~2413894,48987206~726746,49928767~4422696,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,130028920~8048538,147675793~3299225,167226701~1958761,172643947~11507262,192928176~5088291,202003294~9052059,225502243~4133844,231065968~17409844,255387586~7771796,266513173~7563069,275274515~8219976,288050963~15249772,312922432~13952494,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,366062574~7797753,375685896~3112271,382830311~404953,385267053~1099561,389497706~7075119,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.546 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [11835209~1716033,16961709~8298723,27215137~984633,30943519~5126877,46468688~2413894,48987206~726746,49928767~4422696,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,130028920~8048538,147675793~3299225,167226701~1958761,172643947~11507262,192928176~5088291,202003294~9052059,225502243~4133844,231065968~17409844,255387586~7771796,266513173~7563069,275274515~8219976,288050963~15249772,312922432~13952494,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,366062574~7797753,375685896~3112271,382830311~404953,385267053~1099561,389497706~7075119,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.554 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [8388608~5162634,16777216~8483216,25717498~2482272,29360128~6710268,46468688~2413894,48987206~726746,49928767~4422696,58720256~4172314,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,100663296~6001242,109051904~6404753,121899613~6928089,130028920~8048538,146800640~4174378,163577856~5607606,171966464~12184745,188743680~9272787,201326592~9728761,222298112~7337975,231065968~17409844,251658240~11501142,264241152~19253339,285968327~17332408,310378496~16496430,331350016~7344960,339738624~5458989,350444844~7313146,360710144~4007447,366062574~7797753,375685896~3112271,382830311~404953,385267053~11305772,405420596~3083115,409582788~931046] 2024-09-13T00:51:20.554 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 9 diff [7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,30943519~5126877,41106664~7775918,48987206~726746,49928767~4422696,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,130028920~8048538,147138413~3836605,167226701~1958761,172643947~11507262,192928176~5088291,202003294~9052059,225502243~4133844,231065968~17409844,255387586~7771796,264304297~9771945,275274515~8219976,288050963~15249772,307269978~110347,308023339~18851587,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,366062574~7797753,375685896~4658742,382830311~404953,385267053~1099561,389497706~7286460,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.554 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,30943519~5126877,41106664~7775918,48987206~726746,49928767~4422696,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,130028920~8048538,147138413~3836605,167226701~1958761,172643947~11507262,192928176~5088291,202003294~9052059,225502243~4133844,231065968~17409844,255387586~7771796,264304297~9771945,275274515~8219976,288050963~15249772,307269978~110347,308023339~18851587,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,366062574~7797753,375685896~4658742,382830311~404953,385267053~1099561,389497706~7286460,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.561 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~9356938,16777216~8483216,25717498~2482272,29360128~6710268,37748736~11133846,48987206~726746,49928767~4422696,58720256~4172314,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,100663296~6001242,109051904~6404753,121899613~6928089,130028920~8048538,146800640~4174378,163577856~5607606,171966464~12184745,188743680~9272787,201326592~9728761,222298112~7337975,231065968~17409844,251658240~11501142,264241152~19253339,285968327~17332408,306184192~20690734,331350016~7344960,339738624~5458989,350444844~7313146,360710144~4007447,366062574~7797753,375685896~4658742,382830311~404953,385267053~11517113,405420596~3083115,409582788~931046] 2024-09-13T00:51:20.561 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 10 diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,30943519~6589017,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,130028920~8048538,147138413~7483873,167226701~1958761,172643947~11507262,185145489~6850225,192928176~5088291,202003294~9052059,218243585~5067439,225502243~4133844,231065968~17409844,249652551~224229,255387586~7771796,264304297~9771945,275274515~8219976,288050963~15249772,307269978~110347,308023339~19566249,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,366062574~7797753,375685896~4658742,382830311~404953,385267053~1099561,389497706~7286460,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.562 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,30943519~6589017,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,62780271~112299,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,130028920~8048538,147138413~7483873,167226701~1958761,172643947~11507262,185145489~6850225,192928176~5088291,202003294~9052059,218243585~5067439,225502243~4133844,231065968~17409844,249652551~224229,255387586~7771796,264304297~9771945,275274515~8219976,288050963~15249772,307269978~110347,308023339~19566249,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,366062574~7797753,375685896~4658742,382830311~404953,385267053~1099561,389497706~7286460,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.568 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~9356938,16777216~8483216,25717498~2482272,29360128~8172408,37748736~11133846,48987206~726746,49928767~4422696,58720256~4172314,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,100663296~6001242,109051904~6404753,121899613~6928089,130028920~8048538,146800640~7821646,163577856~5607606,171966464~12184745,184549376~13467091,201326592~9728761,218243585~11392502,231065968~17409844,249652551~224229,251658240~11501142,264241152~19253339,285968327~17332408,306184192~21405396,331350016~7344960,339738624~5458989,350444844~7313146,360710144~4007447,366062574~7797753,375685896~4658742,382830311~404953,385267053~11517113,405420596~3083115,409582788~931046] 2024-09-13T00:51:20.569 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 11 diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,62780271~112299,64412830~13037272,78422427~5908409,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,130028920~8048538,147138413~7483873,167226701~16924508,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~8219976,288050963~15249772,307269978~110347,308023339~19566249,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,366062574~7797753,375685896~4658742,382830311~404953,385267053~1099561,389497706~7286460,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.569 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,62780271~112299,64412830~13037272,78422427~5908409,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,130028920~8048538,147138413~7483873,167226701~16924508,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~8219976,288050963~15249772,307269978~110347,308023339~19566249,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,366062574~7797753,375685896~4658742,382830311~404953,385267053~1099561,389497706~7286460,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.576 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~9356938,16777216~8483216,25717498~2482272,29360128~8172408,37748736~11133846,48987206~726746,49928767~4422696,58720256~4172314,64412830~19918006,92274688~3369448,100663296~6001242,109051904~6404753,121899613~6928089,130028920~8048538,146800640~7821646,163577856~20573353,184549376~16675211,201326592~9728761,218243585~11392502,231065968~20106320,251658240~11501142,264241152~19253339,285968327~17332408,306184192~21405396,331350016~7344960,339738624~5458989,350444844~7313146,360710144~4007447,366062574~7797753,375685896~4658742,382830311~404953,385267053~11517113,405420596~3083115,409582788~931046] 2024-09-13T00:51:20.576 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 12 diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,62780271~112299,64412830~13037272,78422427~11925226,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,129060064~9017394,147138413~7483873,167226701~16924508,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~8219976,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~4838512,343111568~2086045,348106239~9651751,363850683~866908,366062574~7797753,375685896~4658742,382830311~404953,384302117~3873732,389497706~7286460,406071955~8147490] 2024-09-13T00:51:20.576 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,62780271~112299,64412830~13037272,78422427~11925226,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,129060064~9017394,147138413~7483873,167226701~16924508,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~8219976,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~4838512,343111568~2086045,348106239~9651751,363850683~866908,366062574~7797753,375685896~4658742,382830311~404953,384302117~3873732,389497706~7286460,406071955~8147490] 2024-09-13T00:51:20.584 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~9356938,16777216~8483216,25717498~2482272,29360128~8172408,37748736~11133846,48987206~726746,49928767~4422696,58720256~4172314,64412830~25934823,92274688~3369448,100663296~6001242,109051904~6404753,121899613~16177845,146800640~7821646,163577856~20573353,184549376~16675211,201326592~9728761,218243585~11392502,231065968~20106320,251658240~11501142,264241152~19253339,285968327~17332408,306184192~51573798,360710144~4007447,366062574~7797753,375685896~4658742,382830311~404953,384302117~12482049,405420596~8798849] 2024-09-13T00:51:20.584 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 13 diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,78422427~11925226,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,129060064~9017394,147138413~7483873,167226701~16924508,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~8219976,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,366062574~7797753,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:20.584 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,78422427~11925226,94755463~888673,104204950~2459588,111226631~4230026,121899613~6928089,129060064~9017394,147138413~7483873,167226701~16924508,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~8219976,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,366062574~7797753,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:20.590 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~9356938,16777216~8483216,25717498~2482272,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~25934823,92274688~3369448,100663296~6001242,109051904~6404753,121899613~16177845,146800640~7821646,163577856~20573353,184549376~16675211,201326592~9728761,218243585~11392502,231065968~20106320,251658240~11501142,264241152~19253339,285968327~17332408,306184192~52344083,360710144~4007447,366062574~7797753,375685896~4658742,381207933~15576233,405420596~8798849] 2024-09-13T00:51:20.590 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 14 diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,78422427~12611916,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,121899613~16177845,147138413~7483873,167226701~16924508,185145489~16079098,202003294~9052059,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~8219976,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,366062574~7797753,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:20.590 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,78422427~12611916,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,121899613~16177845,147138413~7483873,167226701~16924508,185145489~16079098,202003294~9052059,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~8219976,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,366062574~7797753,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:20.598 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~9356938,16777216~8483216,25717498~2482272,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~6001242,109051904~6404753,121899613~16177845,146800640~7821646,163577856~20573353,184549376~16675211,201326592~9728761,218243585~11392502,231065968~20106320,251658240~11501142,264241152~19253339,285968327~17332408,306184192~52344083,360710144~4007447,366062574~7797753,375685896~4658742,381207933~15576233,405420596~8798849] 2024-09-13T00:51:20.598 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 15 diff [1533919~9987224,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,121899613~16177845,143476416~11145870,167226701~16924508,185145489~16079098,201287152~9768201,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~8219976,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,366062574~7797753,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.598 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~9987224,11835209~1716033,16961709~8298723,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,121899613~16177845,143476416~11145870,167226701~16924508,185145489~16079098,201287152~9768201,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~8219976,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,366062574~7797753,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:20 smithi088 bash[68960]: cluster 2024-09-13T00:51:18.830799+0000 mgr.y (mgr.25304) 463 : cluster [DBG] pgmap v959: 105 pgs: 105 active+clean; 710 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 232 KiB/s rd, 17 MiB/s wr, 347 op/s 2024-09-13T00:51:20.601 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:20 smithi088 bash[68960]: cluster 2024-09-13T00:51:19.241425+0000 mon.a (mon.0) 1409 : cluster [DBG] osdmap e877: 8 total, 8 up, 8 in 2024-09-13T00:51:20.605 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~13551242,16777216~8483216,25717498~2482272,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~6001242,109051904~6404753,121899613~16177845,142606336~12015950,163577856~20573353,184549376~26505977,218243585~11392502,231065968~20106320,251658240~11501142,264241152~19253339,285968327~17332408,306184192~52344083,360710144~4007447,366062574~7797753,375685896~4658742,381207933~15576233,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.605 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 16 diff [838088~10683055,11835209~1716033,16961709~8960196,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,121899613~16303026,143476416~11145870,158413680~3974333,167226701~16924508,185145489~16079098,201287152~9768201,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~8219976,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,366062574~7797753,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.605 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,16961709~8960196,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,121899613~16303026,143476416~11145870,158413680~3974333,167226701~16924508,185145489~16079098,201287152~9768201,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~8219976,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,366062574~7797753,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.613 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~13551242,16777216~11422554,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~6001242,109051904~6404753,121899613~16303026,142606336~12015950,155189248~7198765,163577856~20573353,184549376~26505977,218243585~11392502,231065968~20106320,251658240~11501142,264241152~19253339,285968327~17332408,306184192~52344083,360710144~4007447,366062574~7797753,375685896~4658742,381207933~15576233,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.613 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 17 diff [838088~10683055,11835209~1716033,16961709~8960196,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~11392502,231065968~20216198,253517969~879309,255387586~7771796,264304297~10571331,275274515~8219976,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.613 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,16961709~8960196,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~11392502,231065968~20216198,253517969~879309,255387586~7771796,264304297~10571331,275274515~8219976,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.620 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~13551242,16777216~11422554,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~6001242,109051904~6404753,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~11392502,231065968~20216198,251658240~11501142,264241152~19253339,285968327~17332408,306184192~52344083,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.620 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 18 diff [838088~10683055,11835209~1716033,16961709~8960196,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~10390811,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.620 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,16961709~8960196,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~10390811,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.627 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~13551242,16777216~11422554,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~21424174,285968327~17332408,306184192~52344083,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.627 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 19 diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.627 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.634 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.634 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 3 diff [104204950~2459588,268623601~2612185,272690745~1385497,275274515~3080302,321663886~5211040,343111568~2086045,363850683~866908,366062574~7797753,385267053~1099561,409582788~931046] 2024-09-13T00:51:20.634 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [104204950~2459588,268623601~2612185,272690745~1385497,275274515~3080302,321663886~5211040,343111568~2086045,363850683~866908,366062574~7797753,385267053~1099561,409582788~931046] 2024-09-13T00:51:20.641 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [100663296~6001242,268435456~2800330,272690745~1385497,275274515~3080302,318767104~8107822,339738624~5458989,360710144~4007447,366062574~7797753,385267053~1099561,409582788~931046] 2024-09-13T00:51:20.641 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 4 diff [16961709~4324172,69362580~8087522,94755463~888673,104204950~2459588,208670672~2384681,261630599~988124,266513173~7563069,275274515~3080302,288350280~12002876,321663886~5211040,333121499~2697631,343111568~2086045,363850683~866908,366062574~7797753,385267053~1099561,391179156~644734,409582788~931046] 2024-09-13T00:51:20.641 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [16961709~4324172,69362580~8087522,94755463~888673,104204950~2459588,208670672~2384681,261630599~988124,266513173~7563069,275274515~3080302,288350280~12002876,321663886~5211040,333121499~2697631,343111568~2086045,363850683~866908,366062574~7797753,385267053~1099561,391179156~644734,409582788~931046] 2024-09-13T00:51:20.648 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [16777216~4508665,69362580~8087522,92274688~3369448,100663296~6001242,208670672~2384681,260046848~2571875,264241152~9835090,275274515~3080302,288350280~12002876,318767104~8107822,331350016~4469114,339738624~5458989,360710144~4007447,366062574~7797753,385267053~1099561,391179156~644734,409582788~931046] 2024-09-13T00:51:20.648 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 5 diff [16961709~4324172,30943519~5126877,69362580~8087522,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,130028920~4587913,147675793~3299225,172643947~3012491,202003294~9052059,239574985~792838,261630599~988124,266513173~7563069,275274515~3080302,288350280~12002876,321663886~5211040,333121499~2697631,343111568~2086045,363850683~866908,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.648 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [16961709~4324172,30943519~5126877,69362580~8087522,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,130028920~4587913,147675793~3299225,172643947~3012491,202003294~9052059,239574985~792838,261630599~988124,266513173~7563069,275274515~3080302,288350280~12002876,321663886~5211040,333121499~2697631,343111568~2086045,363850683~866908,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.655 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [16777216~4508665,30943519~5126877,69362580~8087522,92274688~3369448,100663296~6001242,109051904~6404753,124397052~1978614,130028920~4587913,146800640~4174378,171966464~3689974,201326592~9728761,239075328~1292495,260046848~2571875,264241152~9835090,275274515~3080302,288350280~12002876,318767104~8107822,331350016~4469114,339738624~5458989,360710144~4007447,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,405420596~3083115,409582788~931046] 2024-09-13T00:51:20.655 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 6 diff [16961709~4324172,30943519~5126877,46468688~2413894,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,130028920~7005286,147675793~3299225,172643947~3012491,202003294~9052059,231065968~14680695,261630599~988124,266513173~7563069,275274515~3080302,288350280~12002876,316777936~3606547,321663886~5211040,333121499~2697631,337297297~1397679,343111568~2086045,363850683~866908,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.655 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [16961709~4324172,30943519~5126877,46468688~2413894,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,130028920~7005286,147675793~3299225,172643947~3012491,202003294~9052059,231065968~14680695,261630599~988124,266513173~7563069,275274515~3080302,288350280~12002876,316777936~3606547,321663886~5211040,333121499~2697631,337297297~1397679,343111568~2086045,363850683~866908,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.662 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [16777216~4508665,30943519~5126877,46468688~2413894,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,100663296~6001242,109051904~6404753,124397052~1978614,130028920~7005286,146800640~4174378,171966464~3689974,201326592~9728761,231065968~14680695,260046848~2571875,264241152~9835090,275274515~3080302,288350280~12002876,316777936~10096990,331350016~7344960,339738624~5458989,360710144~4007447,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,405420596~3083115,409582788~931046] 2024-09-13T00:51:20.662 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 7 diff [16961709~4324172,30943519~5126877,46468688~2413894,49928767~4422696,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,130028920~8048538,147675793~3299225,172643947~5798739,192928176~5088291,202003294~9052059,231065968~17409844,256716141~6443241,266513173~7563069,275274515~3080302,288350280~14950455,316777936~10096990,333121499~2697631,337297297~1397679,343111568~2086045,363850683~866908,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.662 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [16961709~4324172,30943519~5126877,46468688~2413894,49928767~4422696,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,130028920~8048538,147675793~3299225,172643947~5798739,192928176~5088291,202003294~9052059,231065968~17409844,256716141~6443241,266513173~7563069,275274515~3080302,288350280~14950455,316777936~10096990,333121499~2697631,337297297~1397679,343111568~2086045,363850683~866908,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.669 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [16777216~4508665,30943519~5126877,46468688~2413894,49928767~4422696,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,100663296~6001242,109051904~6404753,124397052~1978614,130028920~8048538,146800640~4174378,171966464~6476222,188743680~9272787,201326592~9728761,231065968~17409844,255852544~7306838,264241152~9835090,275274515~3080302,288350280~14950455,316777936~10096990,331350016~7344960,339738624~5458989,360710144~4007447,366062574~7797753,382830311~404953,385267053~1099561,391179156~644734,405420596~3083115,409582788~931046] 2024-09-13T00:51:20.669 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 8 diff [16961709~8298723,30943519~5126877,46468688~2413894,48987206~726746,49928767~4422696,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,130028920~8048538,147675793~3299225,172643947~5798739,192928176~5088291,202003294~9052059,225502243~4133844,231065968~17409844,255387586~7771796,266513173~7563069,275274515~3080302,288350280~14950455,316777936~10096990,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,366062574~7797753,375685896~3112271,382830311~404953,385267053~1099561,389497706~7075119,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.669 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [16961709~8298723,30943519~5126877,46468688~2413894,48987206~726746,49928767~4422696,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,130028920~8048538,147675793~3299225,172643947~5798739,192928176~5088291,202003294~9052059,225502243~4133844,231065968~17409844,255387586~7771796,266513173~7563069,275274515~3080302,288350280~14950455,316777936~10096990,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,366062574~7797753,375685896~3112271,382830311~404953,385267053~1099561,389497706~7075119,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.676 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [16777216~8483216,30943519~5126877,46468688~2413894,48987206~726746,49928767~4422696,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,100663296~6001242,109051904~6404753,124397052~1978614,130028920~8048538,146800640~4174378,171966464~6476222,188743680~9272787,201326592~9728761,222298112~7337975,231065968~17409844,251658240~11501142,264241152~9835090,275274515~3080302,288350280~14950455,316777936~10096990,331350016~7344960,339738624~5458989,350444844~7313146,360710144~4007447,366062574~7797753,375685896~3112271,382830311~404953,385267053~11305772,405420596~3083115,409582788~931046] 2024-09-13T00:51:20.676 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 9 diff [7447179~4073964,16961709~8298723,30943519~5126877,41106664~7775918,48987206~726746,49928767~4422696,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,130028920~8048538,147138413~3836605,172643947~5798739,192928176~5088291,202003294~9052059,225502243~4133844,231065968~17409844,255387586~7771796,264304297~9771945,275274515~3080302,288350280~14950455,307269978~110347,308023339~7288248,316777936~10096990,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,366062574~7797753,375685896~4658742,382830311~404953,385267053~1099561,389497706~7286460,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.676 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [7447179~4073964,16961709~8298723,30943519~5126877,41106664~7775918,48987206~726746,49928767~4422696,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,130028920~8048538,147138413~3836605,172643947~5798739,192928176~5088291,202003294~9052059,225502243~4133844,231065968~17409844,255387586~7771796,264304297~9771945,275274515~3080302,288350280~14950455,307269978~110347,308023339~7288248,316777936~10096990,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,366062574~7797753,375685896~4658742,382830311~404953,385267053~1099561,389497706~7286460,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.683 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~7326839,16777216~8483216,30943519~5126877,37748736~11133846,48987206~726746,49928767~4422696,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,100663296~6001242,109051904~6404753,124397052~1978614,130028920~8048538,146800640~4174378,171966464~6476222,188743680~9272787,201326592~9728761,222298112~7337975,231065968~17409844,251658240~11501142,264241152~9835090,275274515~3080302,288350280~14950455,306184192~9127395,316777936~10096990,331350016~7344960,339738624~5458989,350444844~7313146,360710144~4007447,366062574~7797753,375685896~4658742,382830311~404953,385267053~11517113,405420596~3083115,409582788~931046] 2024-09-13T00:51:20.683 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 10 diff [1533919~394093,7447179~4073964,16961709~8298723,30943519~6589017,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,130028920~8048538,147138413~7483873,172643947~5798739,185145489~6850225,192928176~5088291,202003294~9052059,218243585~5067439,225502243~4133844,231065968~17409844,249652551~224229,255387586~7771796,264304297~9771945,275274515~3080302,288350280~14950455,307269978~110347,308023339~7288248,316777936~10811652,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,366062574~7797753,375685896~4658742,382830311~404953,385267053~1099561,389497706~7286460,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.683 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,16961709~8298723,30943519~6589017,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,130028920~8048538,147138413~7483873,172643947~5798739,185145489~6850225,192928176~5088291,202003294~9052059,218243585~5067439,225502243~4133844,231065968~17409844,249652551~224229,255387586~7771796,264304297~9771945,275274515~3080302,288350280~14950455,307269978~110347,308023339~7288248,316777936~10811652,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,366062574~7797753,375685896~4658742,382830311~404953,385267053~1099561,389497706~7286460,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.690 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,16777216~8483216,30943519~6589017,37748736~11133846,48987206~726746,49928767~4422696,59919134~784701,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,100663296~6001242,109051904~6404753,124397052~1978614,130028920~8048538,146800640~7821646,171966464~6476222,184549376~13467091,201326592~9728761,218243585~11392502,231065968~17409844,249652551~224229,251658240~11501142,264241152~9835090,275274515~3080302,288350280~14950455,306184192~9127395,316777936~10811652,331350016~7344960,339738624~5458989,350444844~7313146,360710144~4007447,366062574~7797753,375685896~4658742,382830311~404953,385267053~11517113,405420596~3083115,409582788~931046] 2024-09-13T00:51:20.691 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 11 diff [1533919~394093,7447179~4073964,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,64412830~13037272,78422427~5908409,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,130028920~8048538,147138413~7483873,169134757~9307929,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~3080302,288350280~14950455,307269978~110347,308023339~7288248,316777936~10811652,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,366062574~7797753,375685896~4658742,382830311~404953,385267053~1099561,389497706~7286460,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.691 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,64412830~13037272,78422427~5908409,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,130028920~8048538,147138413~7483873,169134757~9307929,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~3080302,288350280~14950455,307269978~110347,308023339~7288248,316777936~10811652,333121499~3214519,337297297~1397679,343111568~2086045,350444844~7313146,363850683~866908,366062574~7797753,375685896~4658742,382830311~404953,385267053~1099561,389497706~7286460,406459211~2044500,409582788~931046] 2024-09-13T00:51:20.698 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,16777216~8483216,29503754~8028782,37748736~11133846,48987206~726746,49928767~4422696,59919134~784701,64412830~19918006,92274688~3369448,100663296~6001242,109051904~6404753,124397052~1978614,130028920~8048538,146800640~7821646,169134757~9307929,184549376~16675211,201326592~9728761,218243585~11392502,231065968~20106320,251658240~11501142,264241152~10634476,275274515~3080302,288350280~14950455,306184192~9127395,316777936~10811652,331350016~7344960,339738624~5458989,350444844~7313146,360710144~4007447,366062574~7797753,375685896~4658742,382830311~404953,385267053~11517113,405420596~3083115,409582788~931046] 2024-09-13T00:51:20.698 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 12 diff [1533919~394093,7447179~4073964,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,64412830~13037272,78422427~11925226,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,129060064~9017394,147138413~7483873,169134757~9307929,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~3080302,288350280~14950455,307269978~110347,308023339~7288248,316777936~15996404,333121499~3214519,336808173~4838512,343111568~2086045,348106239~9651751,363850683~866908,366062574~7797753,375685896~4658742,382830311~404953,384302117~3873732,389497706~7286460,406071955~8147490] 2024-09-13T00:51:20.698 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,64412830~13037272,78422427~11925226,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,129060064~9017394,147138413~7483873,169134757~9307929,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~3080302,288350280~14950455,307269978~110347,308023339~7288248,316777936~15996404,333121499~3214519,336808173~4838512,343111568~2086045,348106239~9651751,363850683~866908,366062574~7797753,375685896~4658742,382830311~404953,384302117~3873732,389497706~7286460,406071955~8147490] 2024-09-13T00:51:20.705 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,16777216~8483216,29503754~8028782,37748736~11133846,48987206~726746,49928767~4422696,59919134~784701,64412830~25934823,92274688~3369448,100663296~6001242,109051904~6404753,124397052~1978614,128827702~9249756,146800640~7821646,169134757~9307929,184549376~16675211,201326592~9728761,218243585~11392502,231065968~20106320,251658240~11501142,264241152~10634476,275274515~3080302,288350280~14950455,306184192~9127395,316777936~40980054,360710144~4007447,366062574~7797753,375685896~4658742,382830311~404953,384302117~12482049,405420596~8798849] 2024-09-13T00:51:20.705 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 13 diff [1533919~394093,7447179~4073964,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,78422427~11925226,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,129060064~9017394,147138413~7483873,169134757~11275754,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~3080302,288350280~14950455,307269978~110347,308023339~7288248,316777936~15996404,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,366062574~7797753,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:20.706 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,78422427~11925226,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,129060064~9017394,147138413~7483873,169134757~11275754,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,275274515~3080302,288350280~14950455,307269978~110347,308023339~7288248,316777936~15996404,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,366062574~7797753,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:20.712 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,16777216~8483216,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~25934823,92274688~3369448,100663296~6001242,109051904~6404753,124397052~1978614,128827702~9249756,146800640~7821646,169134757~11275754,184549376~16675211,201326592~9728761,218243585~11392502,231065968~20106320,251658240~11501142,264241152~10634476,275274515~3080302,288350280~14950455,306184192~9127395,316777936~41750339,360710144~4007447,366062574~7797753,375685896~4658742,381207933~15576233,405420596~8798849] 2024-09-13T00:51:20.712 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 14 diff [1533919~394093,7447179~4073964,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,78422427~12611916,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,126832157~11245301,147138413~7483873,169134757~11275754,185145489~16079098,202003294~9052059,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~3080302,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,366062574~7797753,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:20.713 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,78422427~12611916,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,126832157~11245301,147138413~7483873,169134757~11275754,185145489~16079098,202003294~9052059,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~3080302,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,366062574~7797753,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:20.719 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,16777216~8483216,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~6001242,109051904~6404753,124397052~1978614,126832157~11245301,146800640~7821646,169134757~11275754,184549376~16675211,201326592~9728761,218243585~11392502,231065968~20106320,251658240~11501142,264241152~10634476,275274515~3080302,288350280~14950455,306184192~52344083,360710144~4007447,366062574~7797753,375685896~4658742,381207933~15576233,405420596~8798849] 2024-09-13T00:51:20.719 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 15 diff [1533919~9987224,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,126832157~11245301,143476416~11145870,169134757~11275754,185145489~16079098,201287152~9768201,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~3080302,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,366062574~7797753,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.719 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~9987224,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,126832157~11245301,143476416~11145870,169134757~11275754,185145489~16079098,201287152~9768201,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~3080302,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348106239~10422036,363850683~866908,366062574~7797753,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.727 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,16777216~8483216,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~6001242,109051904~6404753,124397052~1978614,126832157~11245301,142606336~12015950,169134757~11275754,184549376~26505977,218243585~11392502,231065968~20106320,251658240~11501142,264241152~10634476,275274515~3080302,288350280~14950455,306184192~52344083,360710144~4007447,366062574~7797753,375685896~4658742,381207933~15576233,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.727 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 16 diff [838088~10683055,16961709~8960196,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,126832157~11370482,143476416~11145870,158413680~3974333,169134757~11275754,185145489~16079098,201287152~9768201,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~3080302,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,366062574~7797753,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.727 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,16961709~8960196,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,126832157~11370482,143476416~11145870,158413680~3974333,169134757~11275754,185145489~16079098,201287152~9768201,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,275274515~3080302,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,366062574~7797753,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.734 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,16777216~9144689,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~6001242,109051904~6404753,124397052~1978614,126832157~11370482,142606336~12015950,155189248~7198765,169134757~11275754,184549376~26505977,218243585~11392502,231065968~20106320,251658240~11501142,264241152~10634476,275274515~3080302,288350280~14950455,306184192~52344083,360710144~4007447,366062574~7797753,375685896~4658742,381207933~15576233,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.735 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 17 diff [838088~10683055,16961709~8960196,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~11392502,231065968~20216198,253517969~879309,255387586~7771796,264304297~10571331,275274515~3412101,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.735 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,16961709~8960196,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~2459588,111226631~4230026,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~11392502,231065968~20216198,253517969~879309,255387586~7771796,264304297~10571331,275274515~3412101,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.742 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,16777216~9144689,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~6001242,109051904~6404753,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~11392502,231065968~20216198,251658240~11501142,264241152~10634476,275274515~3412101,288350280~14950455,306184192~52344083,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.742 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 18 diff [838088~10683055,16961709~8960196,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~10390811,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.742 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,16961709~8960196,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~10390811,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,348003072~10525203,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.749 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,16777216~9144689,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~10390811,288350280~14950455,306184192~52344083,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.749 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 19 diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.749 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.757 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.757 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 4 diff [16961709~4324172,69362580~8087522,94755463~888673,208670672~2384681,261630599~988124,266513173~7170792,288350280~12002876,333121499~2697631,391179156~644734] 2024-09-13T00:51:20.757 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [16961709~4324172,69362580~8087522,94755463~888673,208670672~2384681,261630599~988124,266513173~7170792,288350280~12002876,333121499~2697631,391179156~644734] 2024-09-13T00:51:20.763 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [16777216~4508665,69362580~8087522,92274688~3369448,208670672~2384681,260046848~2571875,264241152~9442813,288350280~12002876,331350016~4469114,391179156~644734] 2024-09-13T00:51:20.764 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 5 diff [16961709~4324172,30943519~5126877,69362580~8087522,94755463~888673,111226631~4230026,124397052~1978614,130028920~4587913,147675793~3299225,172643947~3012491,202003294~9052059,239574985~792838,261630599~988124,266513173~7170792,288350280~12002876,333121499~2697631,382830311~404953,391179156~644734,406459211~2044500] 2024-09-13T00:51:20.764 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [16961709~4324172,30943519~5126877,69362580~8087522,94755463~888673,111226631~4230026,124397052~1978614,130028920~4587913,147675793~3299225,172643947~3012491,202003294~9052059,239574985~792838,261630599~988124,266513173~7170792,288350280~12002876,333121499~2697631,382830311~404953,391179156~644734,406459211~2044500] 2024-09-13T00:51:20.770 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [16777216~4508665,30943519~5126877,69362580~8087522,92274688~3369448,109051904~6404753,124397052~1978614,130028920~4587913,146800640~4174378,171966464~3689974,201326592~9728761,239075328~1292495,260046848~2571875,264241152~9442813,288350280~12002876,331350016~4469114,382830311~404953,391179156~644734,405420596~3083115] 2024-09-13T00:51:20.771 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 6 diff [16961709~4324172,30943519~5126877,46468688~2413894,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,111226631~4230026,124397052~1978614,130028920~7005286,147675793~3299225,172643947~3012491,202003294~9052059,231065968~14680695,261630599~988124,266513173~7170792,288350280~12002876,316777936~3606547,333121499~2697631,337297297~1397679,382830311~404953,391179156~644734,406459211~2044500] 2024-09-13T00:51:20.771 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [16961709~4324172,30943519~5126877,46468688~2413894,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,111226631~4230026,124397052~1978614,130028920~7005286,147675793~3299225,172643947~3012491,202003294~9052059,231065968~14680695,261630599~988124,266513173~7170792,288350280~12002876,316777936~3606547,333121499~2697631,337297297~1397679,382830311~404953,391179156~644734,406459211~2044500] 2024-09-13T00:51:20.778 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [16777216~4508665,30943519~5126877,46468688~2413894,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,109051904~6404753,124397052~1978614,130028920~7005286,146800640~4174378,171966464~3689974,201326592~9728761,231065968~14680695,260046848~2571875,264241152~9442813,288350280~12002876,316777936~3606547,331350016~7344960,382830311~404953,391179156~644734,405420596~3083115] 2024-09-13T00:51:20.778 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 7 diff [16961709~4324172,30943519~5126877,46468688~2413894,49928767~4422696,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,111226631~4230026,124397052~1978614,130028920~8048538,147675793~3299225,172643947~5798739,192928176~5088291,202003294~9052059,231065968~17409844,256716141~6443241,266513173~7170792,288350280~14950455,316777936~6007349,333121499~2697631,337297297~1397679,368358100~1519512,382830311~404953,391179156~644734,406459211~2044500] 2024-09-13T00:51:20.778 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [16961709~4324172,30943519~5126877,46468688~2413894,49928767~4422696,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,111226631~4230026,124397052~1978614,130028920~8048538,147675793~3299225,172643947~5798739,192928176~5088291,202003294~9052059,231065968~17409844,256716141~6443241,266513173~7170792,288350280~14950455,316777936~6007349,333121499~2697631,337297297~1397679,368358100~1519512,382830311~404953,391179156~644734,406459211~2044500] 2024-09-13T00:51:20.785 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [16777216~4508665,30943519~5126877,46468688~2413894,49928767~4422696,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,109051904~6404753,124397052~1978614,130028920~8048538,146800640~4174378,171966464~6476222,188743680~9272787,201326592~9728761,231065968~17409844,255852544~7306838,264241152~9442813,288350280~14950455,316777936~6007349,331350016~7344960,368358100~1519512,382830311~404953,391179156~644734,405420596~3083115] 2024-09-13T00:51:20.785 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 8 diff [16961709~8298723,30943519~5126877,46468688~2413894,48987206~726746,49928767~4422696,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,111226631~4230026,124397052~1978614,130028920~8048538,147675793~3299225,172643947~5798739,192928176~5088291,202003294~9052059,225502243~4133844,231065968~17409844,255387586~7771796,266513173~7170792,288350280~14950455,316777936~6007349,333121499~3214519,337297297~1397679,350444844~7313146,368358100~1519512,375685896~3112271,382830311~404953,389497706~7075119,406459211~2044500] 2024-09-13T00:51:20.785 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [16961709~8298723,30943519~5126877,46468688~2413894,48987206~726746,49928767~4422696,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,111226631~4230026,124397052~1978614,130028920~8048538,147675793~3299225,172643947~5798739,192928176~5088291,202003294~9052059,225502243~4133844,231065968~17409844,255387586~7771796,266513173~7170792,288350280~14950455,316777936~6007349,333121499~3214519,337297297~1397679,350444844~7313146,368358100~1519512,375685896~3112271,382830311~404953,389497706~7075119,406459211~2044500] 2024-09-13T00:51:20.791 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [16777216~8483216,30943519~5126877,46468688~2413894,48987206~726746,49928767~4422696,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,109051904~6404753,124397052~1978614,130028920~8048538,146800640~4174378,171966464~6476222,188743680~9272787,201326592~9728761,222298112~7337975,231065968~17409844,251658240~11501142,264241152~9442813,288350280~14950455,316777936~6007349,331350016~7344960,350444844~7313146,368358100~1519512,375685896~3112271,382830311~404953,386366614~10206211,405420596~3083115] 2024-09-13T00:51:20.791 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 9 diff [7447179~4073964,16961709~8298723,30943519~5126877,41106664~7775918,48987206~726746,49928767~4422696,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,111226631~4230026,124397052~1978614,130028920~8048538,147138413~3836605,172643947~5798739,192928176~5088291,202003294~9052059,225502243~4133844,231065968~17409844,255387586~7771796,264304297~9379668,288350280~14950455,307269978~110347,308023339~7288248,316777936~6007349,333121499~3214519,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,389497706~7286460,406459211~2044500] 2024-09-13T00:51:20.792 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [7447179~4073964,16961709~8298723,30943519~5126877,41106664~7775918,48987206~726746,49928767~4422696,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,111226631~4230026,124397052~1978614,130028920~8048538,147138413~3836605,172643947~5798739,192928176~5088291,202003294~9052059,225502243~4133844,231065968~17409844,255387586~7771796,264304297~9379668,288350280~14950455,307269978~110347,308023339~7288248,316777936~6007349,333121499~3214519,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,389497706~7286460,406459211~2044500] 2024-09-13T00:51:20.799 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~7326839,16777216~8483216,30943519~5126877,37748736~11133846,48987206~726746,49928767~4422696,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,109051904~6404753,124397052~1978614,130028920~8048538,146800640~4174378,171966464~6476222,188743680~9272787,201326592~9728761,222298112~7337975,231065968~17409844,251658240~11501142,264241152~9442813,288350280~14950455,306184192~9127395,316777936~6007349,331350016~7344960,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,386366614~10417552,405420596~3083115] 2024-09-13T00:51:20.799 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 10 diff [1533919~394093,7447179~4073964,16961709~8298723,30943519~6589017,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,111226631~4230026,124397052~1978614,130028920~8048538,147138413~7483873,172643947~5798739,185145489~6850225,192928176~5088291,202003294~9052059,218243585~5067439,225502243~4133844,231065968~17409844,249652551~224229,255387586~7771796,264304297~9379668,288350280~14950455,307269978~110347,308023339~7288248,316777936~6007349,324279722~3309866,333121499~3214519,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,389497706~7286460,406459211~2044500] 2024-09-13T00:51:20.799 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,16961709~8298723,30943519~6589017,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,67930859~945201,69006533~8443569,81411365~2058558,94755463~888673,111226631~4230026,124397052~1978614,130028920~8048538,147138413~7483873,172643947~5798739,185145489~6850225,192928176~5088291,202003294~9052059,218243585~5067439,225502243~4133844,231065968~17409844,249652551~224229,255387586~7771796,264304297~9379668,288350280~14950455,307269978~110347,308023339~7288248,316777936~6007349,324279722~3309866,333121499~3214519,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,389497706~7286460,406459211~2044500] 2024-09-13T00:51:20.806 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,16777216~8483216,30943519~6589017,37748736~11133846,48987206~726746,49928767~4422696,59919134~784701,67930859~945201,69006533~8443569,79691776~3778147,92274688~3369448,109051904~6404753,124397052~1978614,130028920~8048538,146800640~7821646,171966464~6476222,184549376~13467091,201326592~9728761,218243585~11392502,231065968~17409844,249652551~224229,251658240~11501142,264241152~9442813,288350280~14950455,306184192~9127395,316777936~6007349,324279722~3309866,331350016~7344960,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,386366614~10417552,405420596~3083115] 2024-09-13T00:51:20.807 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 11 diff [1533919~394093,7447179~4073964,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,64412830~13037272,78422427~5908409,94755463~888673,111226631~4230026,124397052~1978614,130028920~8048538,147138413~7483873,169134757~9307929,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,288350280~14950455,307269978~110347,308023339~7288248,316777936~6007349,324279722~3309866,333121499~3214519,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,389497706~7286460,406459211~2044500] 2024-09-13T00:51:20.807 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,64412830~13037272,78422427~5908409,94755463~888673,111226631~4230026,124397052~1978614,130028920~8048538,147138413~7483873,169134757~9307929,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,288350280~14950455,307269978~110347,308023339~7288248,316777936~6007349,324279722~3309866,333121499~3214519,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,389497706~7286460,406459211~2044500] 2024-09-13T00:51:20.813 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,16777216~8483216,29503754~8028782,37748736~11133846,48987206~726746,49928767~4422696,59919134~784701,64412830~19918006,92274688~3369448,109051904~6404753,124397052~1978614,130028920~8048538,146800640~7821646,169134757~9307929,184549376~16675211,201326592~9728761,218243585~11392502,231065968~20106320,251658240~11501142,264241152~10634476,288350280~14950455,306184192~9127395,316777936~6007349,324279722~3309866,331350016~7344960,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,386366614~10417552,405420596~3083115] 2024-09-13T00:51:20.813 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 12 diff [1533919~394093,7447179~4073964,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,64412830~13037272,78422427~11925226,94755463~888673,111226631~4230026,124397052~1978614,129060064~9017394,147138413~7483873,169134757~9307929,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,288350280~14950455,307269978~110347,308023339~7288248,316777936~6007349,324279722~8494618,333121499~3214519,336808173~4838512,348106239~9651751,368358100~1519512,375685896~4658742,382830311~404953,384302117~3873732,389497706~7286460,406071955~8147490] 2024-09-13T00:51:20.813 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,64412830~13037272,78422427~11925226,94755463~888673,111226631~4230026,124397052~1978614,129060064~9017394,147138413~7483873,169134757~9307929,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,288350280~14950455,307269978~110347,308023339~7288248,316777936~6007349,324279722~8494618,333121499~3214519,336808173~4838512,348106239~9651751,368358100~1519512,375685896~4658742,382830311~404953,384302117~3873732,389497706~7286460,406071955~8147490] 2024-09-13T00:51:20.820 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,16777216~8483216,29503754~8028782,37748736~11133846,48987206~726746,49928767~4422696,59919134~784701,64412830~25934823,92274688~3369448,109051904~6404753,124397052~1978614,128827702~9249756,146800640~7821646,169134757~9307929,184549376~16675211,201326592~9728761,218243585~11392502,231065968~20106320,251658240~11501142,264241152~10634476,288350280~14950455,306184192~9127395,316777936~6007349,324279722~17366963,345197613~12560377,368358100~1519512,375685896~4658742,382830311~404953,384302117~12482049,405420596~8798849] 2024-09-13T00:51:20.820 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 13 diff [1533919~394093,7447179~4073964,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,78422427~11925226,94755463~888673,111226631~4230026,124397052~1978614,129060064~9017394,147138413~7483873,169134757~11275754,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,288350280~14950455,307269978~110347,308023339~7288248,316777936~6007349,324279722~8494618,333121499~3214519,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:20.820 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,78422427~11925226,94755463~888673,111226631~4230026,124397052~1978614,129060064~9017394,147138413~7483873,169134757~11275754,185145489~6850225,192928176~8296411,202003294~9052059,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~10571331,288350280~14950455,307269978~110347,308023339~7288248,316777936~6007349,324279722~8494618,333121499~3214519,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:20.827 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,16777216~8483216,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~25934823,92274688~3369448,109051904~6404753,124397052~1978614,128827702~9249756,146800640~7821646,169134757~11275754,184549376~16675211,201326592~9728761,218243585~11392502,231065968~20106320,251658240~11501142,264241152~10634476,288350280~14950455,306184192~9127395,316777936~6007349,324279722~20803320,345197613~13330662,368358100~1519512,375685896~4658742,381207933~15576233,405420596~8798849] 2024-09-13T00:51:20.827 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 14 diff [1533919~394093,7447179~4073964,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,78422427~12611916,94079298~566015,94755463~888673,111226631~4230026,124397052~1978614,126832157~11245301,147138413~7483873,169134757~11275754,185145489~16079098,202003294~9052059,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,288350280~14950455,307269978~110347,308023339~14761946,324279722~8494618,333121499~3214519,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:20.827 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,78422427~12611916,94079298~566015,94755463~888673,111226631~4230026,124397052~1978614,126832157~11245301,147138413~7483873,169134757~11275754,185145489~16079098,202003294~9052059,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,288350280~14950455,307269978~110347,308023339~14761946,324279722~8494618,333121499~3214519,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:20.834 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,16777216~8483216,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,109051904~6404753,124397052~1978614,126832157~11245301,146800640~7821646,169134757~11275754,184549376~16675211,201326592~9728761,218243585~11392502,231065968~20106320,251658240~11501142,264241152~10634476,288350280~14950455,306184192~16601093,324279722~20803320,345197613~13330662,368358100~1519512,375685896~4658742,381207933~15576233,405420596~8798849] 2024-09-13T00:51:20.834 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 15 diff [1533919~9987224,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,111226631~4230026,124397052~1978614,126832157~11245301,143476416~11145870,169134757~11275754,185145489~16079098,201287152~9768201,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,288350280~14950455,307269978~110347,308023339~14761946,324279722~8494618,333121499~3214519,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.835 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~9987224,16961709~8298723,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,111226631~4230026,124397052~1978614,126832157~11245301,143476416~11145870,169134757~11275754,185145489~16079098,201287152~9768201,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,288350280~14950455,307269978~110347,308023339~14761946,324279722~8494618,333121499~3214519,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.842 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,16777216~8483216,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,109051904~6404753,124397052~1978614,126832157~11245301,142606336~12015950,169134757~11275754,184549376~26505977,218243585~11392502,231065968~20106320,251658240~11501142,264241152~10634476,288350280~14950455,306184192~16601093,324279722~20803320,345197613~13330662,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.842 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 16 diff [838088~10683055,16961709~8960196,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,111226631~4230026,124397052~1978614,126832157~11370482,143476416~11145870,158413680~3974333,169134757~11275754,185145489~16079098,201287152~9768201,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,288350280~14950455,307269978~110347,308023339~14761946,324279722~8494618,333121499~3214519,336808173~8274869,348003072~10525203,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.842 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,16961709~8960196,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,111226631~4230026,124397052~1978614,126832157~11370482,143476416~11145870,158413680~3974333,169134757~11275754,185145489~16079098,201287152~9768201,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,288350280~14950455,307269978~110347,308023339~14761946,324279722~8494618,333121499~3214519,336808173~8274869,348003072~10525203,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.849 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,16777216~9144689,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,109051904~6404753,124397052~1978614,126832157~11370482,142606336~12015950,155189248~7198765,169134757~11275754,184549376~26505977,218243585~11392502,231065968~20106320,251658240~11501142,264241152~10634476,288350280~14950455,306184192~16601093,324279722~20803320,345197613~13330662,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.849 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 17 diff [838088~10683055,16961709~8960196,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,111226631~4230026,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~11392502,231065968~20216198,253517969~879309,255387586~7771796,264304297~10571331,276169467~2517149,288350280~14950455,307269978~110347,308023339~14761946,324279722~8494618,333121499~3214519,336808173~8274869,348003072~10525203,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.849 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,16961709~8960196,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,111226631~4230026,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~11392502,231065968~20216198,253517969~879309,255387586~7771796,264304297~10571331,276169467~2517149,288350280~14950455,307269978~110347,308023339~14761946,324279722~8494618,333121499~3214519,336808173~8274869,348003072~10525203,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.856 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,16777216~9144689,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,109051904~6404753,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~11392502,231065968~20216198,251658240~11501142,264241152~10634476,276169467~2517149,288350280~14950455,306184192~16601093,324279722~20803320,345197613~13330662,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.856 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 18 diff [838088~10683055,16961709~8960196,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~9495859,288350280~14950455,307269978~110347,308023339~14761946,324279722~8494618,333121499~3214519,336808173~8274869,348003072~10525203,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.856 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,16961709~8960196,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~9495859,288350280~14950455,307269978~110347,308023339~14761946,324279722~8494618,333121499~3214519,336808173~8274869,348003072~10525203,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.863 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,16777216~9144689,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~9495859,288350280~14950455,306184192~16601093,324279722~20803320,345197613~13330662,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.863 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 19 diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.863 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.870 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~10489226,288350280~14950455,306184192~38898850,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.870 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 5 diff [30943519~5126877,111226631~4230026,124397052~1978614,130028920~4587913,147675793~3299225,172643947~3012491,202003294~7354786,239574985~792838,382830311~404953,406459211~2044500] 2024-09-13T00:51:20.870 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [30943519~5126877,111226631~4230026,124397052~1978614,130028920~4587913,147675793~3299225,172643947~3012491,202003294~7354786,239574985~792838,382830311~404953,406459211~2044500] 2024-09-13T00:51:20.877 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [30943519~5126877,109051904~6404753,124397052~1978614,130028920~4587913,146800640~4174378,171966464~3689974,201326592~8031488,239075328~1292495,382830311~404953,405420596~3083115] 2024-09-13T00:51:20.877 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 6 diff [30943519~5126877,46468688~2413894,67930859~945201,69006533~4513357,81411365~2058558,111226631~4230026,124397052~1978614,130028920~7005286,147675793~3299225,172643947~3012491,202003294~7354786,231065968~14680695,316777936~3606547,337297297~1397679,382830311~404953,406459211~2044500] 2024-09-13T00:51:20.877 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [30943519~5126877,46468688~2413894,67930859~945201,69006533~4513357,81411365~2058558,111226631~4230026,124397052~1978614,130028920~7005286,147675793~3299225,172643947~3012491,202003294~7354786,231065968~14680695,316777936~3606547,337297297~1397679,382830311~404953,406459211~2044500] 2024-09-13T00:51:20.884 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [30943519~5126877,46468688~2413894,67930859~945201,69006533~4513357,79691776~3778147,109051904~6404753,124397052~1978614,130028920~7005286,146800640~4174378,171966464~3689974,201326592~8031488,231065968~14680695,316777936~3606547,335819130~2875846,382830311~404953,405420596~3083115] 2024-09-13T00:51:20.884 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 7 diff [30943519~5126877,46468688~2413894,49928767~4422696,67930859~945201,69006533~4513357,81411365~2058558,111226631~4230026,124397052~1978614,130028920~8048538,147675793~3299225,172643947~5798739,192928176~5088291,202003294~7354786,231065968~17409844,256716141~6443241,297925649~5375086,316777936~6007349,337297297~1397679,368358100~1519512,382830311~404953,406459211~2044500] 2024-09-13T00:51:20.884 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [30943519~5126877,46468688~2413894,49928767~4422696,67930859~945201,69006533~4513357,81411365~2058558,111226631~4230026,124397052~1978614,130028920~8048538,147675793~3299225,172643947~5798739,192928176~5088291,202003294~7354786,231065968~17409844,256716141~6443241,297925649~5375086,316777936~6007349,337297297~1397679,368358100~1519512,382830311~404953,406459211~2044500] 2024-09-13T00:51:20.891 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [30943519~5126877,46468688~2413894,49928767~4422696,67930859~945201,69006533~4513357,79691776~3778147,109051904~6404753,124397052~1978614,130028920~8048538,146800640~4174378,171966464~6476222,188743680~9272787,201326592~8031488,231065968~17409844,255852544~7306838,297925649~5375086,316777936~6007349,335819130~2875846,368358100~1519512,382830311~404953,405420596~3083115] 2024-09-13T00:51:20.891 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 8 diff [20041909~5218523,30943519~5126877,46468688~2413894,48987206~726746,49928767~4422696,67930859~945201,69006533~4513357,81411365~2058558,111226631~4230026,124397052~1978614,130028920~8048538,147675793~3299225,172643947~5798739,192928176~5088291,202003294~7354786,225502243~4133844,231065968~17409844,255387586~7771796,295560575~7740160,316777936~6007349,335285595~1050423,337297297~1397679,350444844~7313146,368358100~1519512,375685896~3112271,382830311~404953,389497706~7075119,406459211~2044500] 2024-09-13T00:51:20.891 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [20041909~5218523,30943519~5126877,46468688~2413894,48987206~726746,49928767~4422696,67930859~945201,69006533~4513357,81411365~2058558,111226631~4230026,124397052~1978614,130028920~8048538,147675793~3299225,172643947~5798739,192928176~5088291,202003294~7354786,225502243~4133844,231065968~17409844,255387586~7771796,295560575~7740160,316777936~6007349,335285595~1050423,337297297~1397679,350444844~7313146,368358100~1519512,375685896~3112271,382830311~404953,389497706~7075119,406459211~2044500] 2024-09-13T00:51:20.898 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [20041909~5218523,30943519~5126877,46468688~2413894,48987206~726746,49928767~4422696,67930859~945201,69006533~4513357,79691776~3778147,109051904~6404753,124397052~1978614,130028920~8048538,146800640~4174378,171966464~6476222,188743680~9272787,201326592~8031488,222298112~7337975,231065968~17409844,251658240~11501142,295560575~7740160,316777936~6007349,335285595~3409381,350444844~7313146,368358100~1519512,375685896~3112271,382830311~404953,386366614~10206211,405420596~3083115] 2024-09-13T00:51:20.898 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 9 diff [7447179~4073964,20041909~5218523,30943519~5126877,41106664~7775918,48987206~726746,49928767~4422696,67930859~945201,69006533~4513357,81411365~2058558,111226631~4230026,124397052~1978614,130028920~8048538,147138413~3836605,172643947~5798739,192928176~5088291,202003294~7354786,225502243~4133844,231065968~17409844,255387586~7771796,264304297~6430095,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,335285595~1050423,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,389497706~7286460,406459211~2044500] 2024-09-13T00:51:20.898 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [7447179~4073964,20041909~5218523,30943519~5126877,41106664~7775918,48987206~726746,49928767~4422696,67930859~945201,69006533~4513357,81411365~2058558,111226631~4230026,124397052~1978614,130028920~8048538,147138413~3836605,172643947~5798739,192928176~5088291,202003294~7354786,225502243~4133844,231065968~17409844,255387586~7771796,264304297~6430095,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,335285595~1050423,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,389497706~7286460,406459211~2044500] 2024-09-13T00:51:20.905 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~7326839,20041909~5218523,30943519~5126877,37748736~11133846,48987206~726746,49928767~4422696,67930859~945201,69006533~4513357,79691776~3778147,109051904~6404753,124397052~1978614,130028920~8048538,146800640~4174378,171966464~6476222,188743680~9272787,201326592~8031488,222298112~7337975,231065968~17409844,251658240~11501142,264304297~6430095,295560575~7740160,306184192~9127395,316777936~6007349,335285595~3409381,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,386366614~10417552,405420596~3083115] 2024-09-13T00:51:20.905 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 10 diff [1533919~394093,7447179~4073964,20041909~5218523,30943519~6589017,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,67930859~945201,69006533~4513357,81411365~2058558,111226631~4230026,124397052~1978614,130028920~8048538,147138413~7483873,172643947~5798739,185145489~6850225,192928176~5088291,202003294~7354786,218243585~5067439,225502243~4133844,231065968~17409844,249652551~224229,255387586~7771796,264304297~6430095,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,324279722~3309866,335285595~1050423,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,389497706~7286460,406459211~2044500] 2024-09-13T00:51:20.905 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20041909~5218523,30943519~6589017,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,67930859~945201,69006533~4513357,81411365~2058558,111226631~4230026,124397052~1978614,130028920~8048538,147138413~7483873,172643947~5798739,185145489~6850225,192928176~5088291,202003294~7354786,218243585~5067439,225502243~4133844,231065968~17409844,249652551~224229,255387586~7771796,264304297~6430095,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,324279722~3309866,335285595~1050423,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,389497706~7286460,406459211~2044500] 2024-09-13T00:51:20.912 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20041909~5218523,30943519~6589017,37748736~11133846,48987206~726746,49928767~4422696,59919134~784701,67930859~945201,69006533~4513357,79691776~3778147,109051904~6404753,124397052~1978614,130028920~8048538,146800640~7821646,171966464~6476222,184549376~13467091,201326592~8031488,218243585~11392502,231065968~17409844,249652551~224229,251658240~11501142,264304297~6430095,295560575~7740160,306184192~9127395,316777936~6007349,324279722~3309866,335285595~3409381,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,386366614~10417552,405420596~3083115] 2024-09-13T00:51:20.912 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 11 diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,64412830~9107060,78422427~5908409,111226631~4230026,124397052~1978614,130028920~8048538,147138413~7483873,169134757~9307929,185145489~6850225,192928176~8296411,202003294~7354786,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,324279722~3309866,335206066~1129952,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,389497706~7286460,406459211~2044500] 2024-09-13T00:51:20.912 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,64412830~9107060,78422427~5908409,111226631~4230026,124397052~1978614,130028920~8048538,147138413~7483873,169134757~9307929,185145489~6850225,192928176~8296411,202003294~7354786,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,324279722~3309866,335206066~1129952,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,389497706~7286460,406459211~2044500] 2024-09-13T00:51:20.919 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~5230633,29503754~8028782,37748736~11133846,48987206~726746,49928767~4422696,59919134~784701,64412830~9107060,77450102~6880734,109051904~6404753,124397052~1978614,130028920~8048538,146800640~7821646,169134757~9307929,184549376~16675211,201326592~8031488,218243585~11392502,231065968~20106320,251658240~11501142,264304297~6430095,273613045~1262583,295560575~7740160,306184192~9127395,316777936~6007349,324279722~3309866,335206066~3488910,350444844~7313146,368358100~1519512,375685896~4658742,382830311~404953,386366614~10417552,405420596~3083115] 2024-09-13T00:51:20.919 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 12 diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,64412830~9107060,78422427~11925226,111226631~4230026,124397052~1978614,129060064~9017394,147138413~7483873,169134757~9307929,185145489~6850225,192928176~8296411,202003294~7354786,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,324279722~8494618,335206066~1129952,336808173~4838512,348106239~9651751,368358100~1519512,375685896~4658742,382830311~404953,384302117~3873732,389497706~7286460,406071955~8147490] 2024-09-13T00:51:20.919 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,64412830~9107060,78422427~11925226,111226631~4230026,124397052~1978614,129060064~9017394,147138413~7483873,169134757~9307929,185145489~6850225,192928176~8296411,202003294~7354786,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,324279722~8494618,335206066~1129952,336808173~4838512,348106239~9651751,368358100~1519512,375685896~4658742,382830311~404953,384302117~3873732,389497706~7286460,406071955~8147490] 2024-09-13T00:51:20.926 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~5230633,29503754~8028782,37748736~11133846,48987206~726746,49928767~4422696,59919134~784701,64412830~9107060,77450102~12897551,109051904~6404753,124397052~1978614,128827702~9249756,146800640~7821646,169134757~9307929,184549376~16675211,201326592~8031488,218243585~11392502,231065968~20106320,251658240~11501142,264304297~6430095,273613045~1262583,295560575~7740160,306184192~9127395,316777936~6007349,324279722~8494618,335206066~6440619,345197613~12560377,368358100~1519512,375685896~4658742,382830311~404953,384302117~12482049,405420596~8798849] 2024-09-13T00:51:20.927 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 13 diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,78422427~11925226,111226631~4230026,124397052~1978614,129060064~9017394,147138413~7483873,169134757~11275754,185145489~6850225,192928176~8296411,202003294~7354786,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:20.927 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,78422427~11925226,111226631~4230026,124397052~1978614,129060064~9017394,147138413~7483873,169134757~11275754,185145489~6850225,192928176~8296411,202003294~7354786,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:20.933 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~5230633,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~12897551,109051904~6404753,124397052~1978614,128827702~9249756,146800640~7821646,169134757~11275754,184549376~16675211,201326592~8031488,218243585~11392502,231065968~20106320,251658240~11501142,264304297~6430095,273613045~1262583,295560575~7740160,306184192~9127395,316777936~6007349,324279722~8494618,335206066~9876976,345197613~13330662,368358100~1519512,375685896~4658742,381207933~15576233,405420596~8798849] 2024-09-13T00:51:20.934 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 14 diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,78422427~12611916,94079298~566015,111226631~4230026,124397052~1978614,126832157~11245301,147138413~7483873,169134757~11275754,185145489~16079098,202003294~7354786,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:20.934 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,78422427~12611916,94079298~566015,111226631~4230026,124397052~1978614,126832157~11245301,147138413~7483873,169134757~11275754,185145489~16079098,202003294~7354786,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:20.940 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~5230633,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,109051904~6404753,124397052~1978614,126832157~11245301,146800640~7821646,169134757~11275754,184549376~16675211,201326592~8031488,218243585~11392502,231065968~20106320,251658240~11501142,264304297~6430095,273613045~1262583,295560575~7740160,306184192~16601093,324279722~8494618,335206066~9876976,345197613~13330662,368358100~1519512,375685896~4658742,381207933~15576233,405420596~8798849] 2024-09-13T00:51:20.941 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 15 diff [1533919~9987224,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,111226631~4230026,124397052~1978614,126832157~11245301,143476416~11145870,169134757~11275754,185145489~16079098,201287152~8070928,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.941 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~9987224,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,111226631~4230026,124397052~1978614,126832157~11245301,143476416~11145870,169134757~11275754,185145489~16079098,201287152~8070928,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.947 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~5230633,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,109051904~6404753,124397052~1978614,126832157~11245301,142606336~12015950,169134757~11275754,184549376~24808704,218243585~11392502,231065968~20106320,251658240~11501142,264304297~10571331,295560575~7740160,306184192~16601093,324279722~8494618,335206066~9876976,345197613~13330662,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.947 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 16 diff [838088~10683055,20029799~5892106,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,111226631~4230026,124397052~1978614,126832157~11370482,143476416~11145870,158413680~3974333,169134757~11275754,185145489~16079098,201287152~8070928,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.947 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,20029799~5892106,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,111226631~4230026,124397052~1978614,126832157~11370482,143476416~11145870,158413680~3974333,169134757~11275754,185145489~16079098,201287152~8070928,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.955 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~5892106,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,109051904~6404753,124397052~1978614,126832157~11370482,142606336~12015950,155189248~7198765,169134757~11275754,184549376~24808704,218243585~11392502,231065968~20106320,251658240~11501142,264304297~10571331,295560575~7740160,306184192~16601093,324279722~8494618,335206066~9876976,345197613~13330662,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.955 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 17 diff [838088~10683055,20029799~5892106,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,111226631~4230026,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~11392502,231065968~20216198,253517969~879309,255387586~7771796,264304297~10571331,276169467~2517149,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.955 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,20029799~5892106,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,111226631~4230026,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~11392502,231065968~20216198,253517969~879309,255387586~7771796,264304297~10571331,276169467~2517149,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.961 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~5892106,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,109051904~6404753,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~11392502,231065968~20216198,251658240~11501142,264304297~10571331,276169467~2517149,295560575~7740160,306184192~16601093,324279722~8494618,335206066~9876976,345197613~13330662,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.962 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 18 diff [838088~10683055,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~9495859,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.962 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~9495859,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.968 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~9495859,295560575~7740160,306184192~16601093,324279722~8494618,335206066~9876976,345197613~13330662,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.968 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 19 diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.968 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:20.976 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:20.976 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 6 diff [46468688~2413894,67930859~945201,69006533~4513357,81411365~2058558,132271796~4762410,231065968~14680695,316777936~3606547,337297297~1397679] 2024-09-13T00:51:20.976 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [46468688~2413894,67930859~945201,69006533~4513357,81411365~2058558,132271796~4762410,231065968~14680695,316777936~3606547,337297297~1397679] 2024-09-13T00:51:20.984 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [46468688~2413894,67930859~945201,69006533~4513357,79691776~3778147,132271796~4762410,231065968~14680695,316777936~3606547,335819130~2875846] 2024-09-13T00:51:20.984 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 7 diff [46468688~2413894,49928767~4422696,67930859~945201,69006533~4513357,81411365~2058558,132271796~5805662,175544318~2898368,192928176~5088291,231065968~17409844,256716141~6443241,297925649~5375086,316777936~6007349,337297297~1397679,368358100~1519512] 2024-09-13T00:51:20.984 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [46468688~2413894,49928767~4422696,67930859~945201,69006533~4513357,81411365~2058558,132271796~5805662,175544318~2898368,192928176~5088291,231065968~17409844,256716141~6443241,297925649~5375086,316777936~6007349,337297297~1397679,368358100~1519512] 2024-09-13T00:51:20.990 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [46468688~2413894,49928767~4422696,67930859~945201,69006533~4513357,79691776~3778147,132271796~5805662,175544318~2898368,188743680~9272787,231065968~17409844,255852544~7306838,297925649~5375086,316777936~6007349,335819130~2875846,368358100~1519512] 2024-09-13T00:51:20.990 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 8 diff [20041909~5218523,46468688~2413894,48987206~726746,49928767~4422696,67930859~945201,69006533~4513357,81411365~2058558,132271796~5805662,175544318~2898368,192928176~5088291,225502243~4133844,231065968~17409844,255387586~7771796,295560575~7740160,316777936~6007349,335285595~1050423,337297297~1397679,350444844~7313146,368358100~1519512,375685896~3112271,389497706~7075119] 2024-09-13T00:51:20.990 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [20041909~5218523,46468688~2413894,48987206~726746,49928767~4422696,67930859~945201,69006533~4513357,81411365~2058558,132271796~5805662,175544318~2898368,192928176~5088291,225502243~4133844,231065968~17409844,255387586~7771796,295560575~7740160,316777936~6007349,335285595~1050423,337297297~1397679,350444844~7313146,368358100~1519512,375685896~3112271,389497706~7075119] 2024-09-13T00:51:20.997 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [20041909~5218523,46468688~2413894,48987206~726746,49928767~4422696,67930859~945201,69006533~4513357,79691776~3778147,132271796~5805662,175544318~2898368,188743680~9272787,222298112~7337975,231065968~17409844,251658240~11501142,295560575~7740160,316777936~6007349,335285595~3409381,350444844~7313146,368358100~1519512,375685896~3112271,386366614~10206211] 2024-09-13T00:51:20.997 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 9 diff [7447179~4073964,20041909~5218523,41106664~7775918,48987206~726746,49928767~4422696,67930859~945201,69006533~4513357,81411365~2058558,132271796~5805662,147138413~922932,175544318~2898368,192928176~5088291,225502243~4133844,231065968~17409844,255387586~7771796,264304297~6430095,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,335285595~1050423,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,389497706~7286460] 2024-09-13T00:51:20.997 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [7447179~4073964,20041909~5218523,41106664~7775918,48987206~726746,49928767~4422696,67930859~945201,69006533~4513357,81411365~2058558,132271796~5805662,147138413~922932,175544318~2898368,192928176~5088291,225502243~4133844,231065968~17409844,255387586~7771796,264304297~6430095,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,335285595~1050423,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.004 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~7326839,20041909~5218523,37748736~11133846,48987206~726746,49928767~4422696,67930859~945201,69006533~4513357,79691776~3778147,132271796~5805662,147138413~922932,175544318~2898368,188743680~9272787,222298112~7337975,231065968~17409844,251658240~11501142,264304297~6430095,295560575~7740160,306184192~9127395,316777936~6007349,335285595~3409381,350444844~7313146,368358100~1519512,375685896~4658742,386366614~10417552] 2024-09-13T00:51:21.004 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 10 diff [1533919~394093,7447179~4073964,20041909~5218523,32783801~4748735,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,67930859~945201,69006533~4513357,81411365~2058558,132271796~5805662,147138413~7483873,175544318~2898368,185145489~6850225,192928176~5088291,218243585~5067439,225502243~4133844,231065968~17409844,249652551~224229,255387586~7771796,264304297~6430095,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,324279722~3309866,335285595~1050423,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.004 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20041909~5218523,32783801~4748735,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,67930859~945201,69006533~4513357,81411365~2058558,132271796~5805662,147138413~7483873,175544318~2898368,185145489~6850225,192928176~5088291,218243585~5067439,225502243~4133844,231065968~17409844,249652551~224229,255387586~7771796,264304297~6430095,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,324279722~3309866,335285595~1050423,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.011 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20041909~5218523,32783801~4748735,37748736~11133846,48987206~726746,49928767~4422696,59919134~784701,67930859~945201,69006533~4513357,79691776~3778147,132271796~5805662,147138413~7483873,175544318~2898368,184549376~13467091,218243585~11392502,231065968~17409844,249652551~224229,251658240~11501142,264304297~6430095,295560575~7740160,306184192~9127395,316777936~6007349,324279722~3309866,335285595~3409381,350444844~7313146,368358100~1519512,375685896~4658742,386366614~10417552] 2024-09-13T00:51:21.012 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 11 diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,64412830~9107060,78422427~5908409,132271796~5805662,147138413~7483873,169134757~5904995,175544318~2898368,185145489~6850225,192928176~8296411,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,324279722~3309866,335206066~1129952,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.012 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,64412830~9107060,78422427~5908409,132271796~5805662,147138413~7483873,169134757~5904995,175544318~2898368,185145489~6850225,192928176~8296411,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,324279722~3309866,335206066~1129952,337297297~1397679,350444844~7313146,368358100~1519512,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.018 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~5230633,29503754~8028782,37748736~11133846,48987206~726746,49928767~4422696,59919134~784701,64412830~9107060,77450102~6880734,132271796~5805662,147138413~7483873,169134757~5904995,175544318~2898368,184549376~16675211,218243585~11392502,231065968~20106320,251658240~11501142,264304297~6430095,273613045~1262583,295560575~7740160,306184192~9127395,316777936~6007349,324279722~3309866,335206066~3488910,350444844~7313146,368358100~1519512,375685896~4658742,386366614~10417552] 2024-09-13T00:51:21.018 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 12 diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,64412830~9107060,78422427~11925226,129060064~9017394,147138413~7483873,169134757~5904995,175544318~2898368,185145489~6850225,192928176~8296411,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,324279722~8494618,335206066~1129952,336808173~4838512,348106239~9651751,368358100~1519512,375685896~4658742,384302117~3873732,389497706~7286460,406071955~8147490] 2024-09-13T00:51:21.018 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~4422696,59919134~784701,64412830~9107060,78422427~11925226,129060064~9017394,147138413~7483873,169134757~5904995,175544318~2898368,185145489~6850225,192928176~8296411,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,324279722~8494618,335206066~1129952,336808173~4838512,348106239~9651751,368358100~1519512,375685896~4658742,384302117~3873732,389497706~7286460,406071955~8147490] 2024-09-13T00:51:21.025 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~5230633,29503754~8028782,37748736~11133846,48987206~726746,49928767~4422696,59919134~784701,64412830~9107060,77450102~12897551,128827702~9249756,147138413~7483873,169134757~5904995,175544318~2898368,184549376~16675211,218243585~11392502,231065968~20106320,251658240~11501142,264304297~6430095,273613045~1262583,295560575~7740160,306184192~9127395,316777936~6007349,324279722~8494618,335206066~6440619,345197613~12560377,368358100~1519512,375685896~4658742,384302117~12482049,406071955~8147490] 2024-09-13T00:51:21.026 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 13 diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,78422427~11925226,129060064~9017394,147138413~7483873,169134757~5904995,175544318~4866193,185145489~6850225,192928176~8296411,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.026 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,78422427~11925226,129060064~9017394,147138413~7483873,169134757~5904995,175544318~4866193,185145489~6850225,192928176~8296411,218243585~5067439,225502243~4133844,231065968~20106320,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,316777936~6007349,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.032 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~5230633,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~12897551,128827702~9249756,147138413~7483873,169134757~5904995,175544318~4866193,184549376~16675211,218243585~11392502,231065968~20106320,251658240~11501142,264304297~6430095,273613045~1262583,295560575~7740160,306184192~9127395,316777936~6007349,324279722~8494618,335206066~9876976,345197613~13330662,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.033 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 14 diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,78422427~12611916,94079298~566015,126832157~11245301,147138413~7483873,169134757~11275754,185145489~16079098,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.033 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,78422427~12611916,94079298~566015,126832157~11245301,147138413~7483873,169134757~11275754,185145489~16079098,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.040 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~5230633,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,126832157~11245301,147138413~7483873,169134757~11275754,184549376~16675211,218243585~11392502,231065968~20106320,251658240~11501142,264304297~6430095,273613045~1262583,295560575~7740160,306184192~16601093,324279722~8494618,335206066~9876976,345197613~13330662,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.041 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 15 diff [1533919~9987224,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,126832157~11245301,143476416~11145870,169134757~11275754,185145489~16079098,201287152~2371737,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.041 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~9987224,20029799~5230633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,126832157~11245301,143476416~11145870,169134757~11275754,185145489~16079098,201287152~2371737,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.047 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~5230633,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,126832157~11245301,142606336~12015950,169134757~11275754,184549376~19109513,218243585~11392502,231065968~20106320,251658240~11501142,264304297~10571331,295560575~7740160,306184192~16601093,324279722~8494618,335206066~9876976,345197613~13330662,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.047 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 16 diff [838088~10683055,20029799~5892106,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,126832157~11370482,143476416~11145870,158413680~3974333,169134757~11275754,185145489~16079098,201287152~2371737,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.047 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,20029799~5892106,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,126832157~11370482,143476416~11145870,158413680~3974333,169134757~11275754,185145489~16079098,201287152~2371737,218243585~11392502,231065968~20106320,253517969~879309,255387586~7771796,264304297~10571331,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.054 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~5892106,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,126832157~11370482,142606336~12015950,155189248~7198765,169134757~11275754,184549376~19109513,218243585~11392502,231065968~20106320,251658240~11501142,264304297~10571331,295560575~7740160,306184192~16601093,324279722~8494618,335206066~9876976,345197613~13330662,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.054 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 17 diff [838088~10683055,20029799~5892106,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~11392502,231065968~20216198,253517969~879309,255387586~7771796,264304297~10571331,276169467~2517149,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.054 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,20029799~5892106,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~11392502,231065968~20216198,253517969~879309,255387586~7771796,264304297~10571331,276169467~2517149,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.061 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~5892106,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~11392502,231065968~20216198,251658240~11501142,264304297~10571331,276169467~2517149,295560575~7740160,306184192~16601093,324279722~8494618,335206066~9876976,345197613~13330662,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.061 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 18 diff [838088~10683055,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~9495859,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.061 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~9495859,295560575~7740160,307269978~110347,308023339~14761946,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.068 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~9495859,295560575~7740160,306184192~16601093,324279722~8494618,335206066~9876976,345197613~13330662,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.069 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 19 diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.069 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.075 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.075 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 7 diff [49928767~4422696,136108718~1968740,175544318~2898368,192928176~5088291,237211987~11263825,256716141~6443241,297925649~5375086,320232816~2552469,368358100~1519512] 2024-09-13T00:51:21.075 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [49928767~4422696,136108718~1968740,175544318~2898368,192928176~5088291,237211987~11263825,256716141~6443241,297925649~5375086,320232816~2552469,368358100~1519512] 2024-09-13T00:51:21.082 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [49928767~4422696,136108718~1968740,175544318~2898368,188743680~9272787,237211987~11263825,255852544~7306838,297925649~5375086,320232816~2552469,368358100~1519512] 2024-09-13T00:51:21.082 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 8 diff [20041909~5218523,48987206~726746,49928767~4422696,136108718~1968740,175544318~2898368,192928176~5088291,225502243~4133844,237211987~11263825,255387586~7771796,295560575~7740160,320232816~2552469,335285595~1050423,350444844~7313146,368358100~1519512,375685896~3112271,389497706~7075119] 2024-09-13T00:51:21.082 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [20041909~5218523,48987206~726746,49928767~4422696,136108718~1968740,175544318~2898368,192928176~5088291,225502243~4133844,237211987~11263825,255387586~7771796,295560575~7740160,320232816~2552469,335285595~1050423,350444844~7313146,368358100~1519512,375685896~3112271,389497706~7075119] 2024-09-13T00:51:21.088 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [20041909~5218523,48987206~726746,49928767~4422696,136108718~1968740,175544318~2898368,188743680~9272787,222298112~7337975,237211987~11263825,251658240~11501142,295560575~7740160,320232816~2552469,335285595~1050423,350444844~7313146,368358100~1519512,375685896~3112271,386366614~10206211] 2024-09-13T00:51:21.089 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 9 diff [7447179~4073964,20041909~5218523,41106664~5965111,48987206~726746,49928767~4422696,136108718~1968740,147138413~922932,175544318~2898368,192928176~5088291,225502243~4133844,237211987~11263825,255387586~7771796,264304297~6430095,295560575~7740160,307269978~110347,308023339~7288248,320232816~2552469,335285595~1050423,350444844~7313146,368358100~1519512,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.089 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [7447179~4073964,20041909~5218523,41106664~5965111,48987206~726746,49928767~4422696,136108718~1968740,147138413~922932,175544318~2898368,192928176~5088291,225502243~4133844,237211987~11263825,255387586~7771796,264304297~6430095,295560575~7740160,307269978~110347,308023339~7288248,320232816~2552469,335285595~1050423,350444844~7313146,368358100~1519512,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.095 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~7326839,20041909~5218523,37748736~9323039,48987206~726746,49928767~4422696,136108718~1968740,147138413~922932,175544318~2898368,188743680~9272787,222298112~7337975,237211987~11263825,251658240~11501142,264304297~6430095,295560575~7740160,306184192~9127395,320232816~2552469,335285595~1050423,350444844~7313146,368358100~1519512,375685896~4658742,386366614~10417552] 2024-09-13T00:51:21.095 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 10 diff [1533919~394093,7447179~4073964,20041909~5218523,32783801~4748735,41106664~5965111,48987206~726746,49928767~4422696,59919134~784701,136108718~1968740,147138413~7483873,175544318~2898368,185145489~6850225,192928176~5088291,218243585~5067439,225502243~4133844,237211987~11263825,249652551~224229,255387586~7771796,264304297~6430095,295560575~7740160,307269978~110347,308023339~7288248,320232816~2552469,324279722~3309866,335285595~1050423,350444844~7313146,368358100~1519512,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.096 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20041909~5218523,32783801~4748735,41106664~5965111,48987206~726746,49928767~4422696,59919134~784701,136108718~1968740,147138413~7483873,175544318~2898368,185145489~6850225,192928176~5088291,218243585~5067439,225502243~4133844,237211987~11263825,249652551~224229,255387586~7771796,264304297~6430095,295560575~7740160,307269978~110347,308023339~7288248,320232816~2552469,324279722~3309866,335285595~1050423,350444844~7313146,368358100~1519512,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.103 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20041909~5218523,32783801~4748735,37748736~9323039,48987206~726746,49928767~4422696,59919134~784701,136108718~1968740,147138413~7483873,175544318~2898368,184549376~13467091,218243585~11392502,237211987~11263825,249652551~224229,251658240~11501142,264304297~6430095,295560575~7740160,306184192~9127395,320232816~2552469,324279722~3309866,335285595~1050423,350444844~7313146,368358100~1519512,375685896~4658742,386366614~10417552] 2024-09-13T00:51:21.103 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 11 diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,49928767~4422696,59919134~784701,64412830~6560139,78422427~5908409,136108718~1968740,147138413~7483873,169134757~5904995,175544318~2898368,185145489~6850225,192928176~8296411,218243585~5067439,225502243~4133844,237211987~13960301,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,320232816~2552469,324279722~3309866,335206066~1129952,350444844~7313146,368358100~1519512,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.103 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,49928767~4422696,59919134~784701,64412830~6560139,78422427~5908409,136108718~1968740,147138413~7483873,169134757~5904995,175544318~2898368,185145489~6850225,192928176~8296411,218243585~5067439,225502243~4133844,237211987~13960301,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,320232816~2552469,324279722~3309866,335206066~1129952,350444844~7313146,368358100~1519512,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.109 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~5230633,29503754~8028782,37748736~9323039,48987206~726746,49928767~4422696,59919134~784701,64412830~6560139,77450102~6880734,136108718~1968740,147138413~7483873,169134757~5904995,175544318~2898368,184549376~16675211,218243585~11392502,237211987~13960301,251658240~11501142,264304297~6430095,273613045~1262583,295560575~7740160,306184192~9127395,320232816~2552469,324279722~3309866,335206066~1129952,350444844~7313146,368358100~1519512,375685896~4658742,386366614~10417552] 2024-09-13T00:51:21.110 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 12 diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,49928767~4422696,59919134~784701,64412830~6560139,78422427~11925226,129060064~5449512,136108718~1968740,147138413~7483873,169134757~5904995,175544318~2898368,185145489~6850225,192928176~8296411,218243585~5067439,225502243~4133844,237211987~13960301,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,320232816~2552469,324279722~8494618,335206066~1129952,336808173~4838512,348106239~9651751,368358100~1519512,375685896~4658742,384302117~3873732,389497706~7286460,406071955~8147490] 2024-09-13T00:51:21.110 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,49928767~4422696,59919134~784701,64412830~6560139,78422427~11925226,129060064~5449512,136108718~1968740,147138413~7483873,169134757~5904995,175544318~2898368,185145489~6850225,192928176~8296411,218243585~5067439,225502243~4133844,237211987~13960301,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,320232816~2552469,324279722~8494618,335206066~1129952,336808173~4838512,348106239~9651751,368358100~1519512,375685896~4658742,384302117~3873732,389497706~7286460,406071955~8147490] 2024-09-13T00:51:21.116 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~5230633,29503754~8028782,37748736~9323039,48987206~726746,49928767~4422696,59919134~784701,64412830~6560139,77450102~12897551,128827702~5681874,136108718~1968740,147138413~7483873,169134757~5904995,175544318~2898368,184549376~16675211,218243585~11392502,237211987~13960301,251658240~11501142,264304297~6430095,273613045~1262583,295560575~7740160,306184192~9127395,320232816~2552469,324279722~8494618,335206066~1129952,336808173~4838512,345197613~12560377,368358100~1519512,375685896~4658742,384302117~12482049,406071955~8147490] 2024-09-13T00:51:21.117 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 13 diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,78422427~11925226,129060064~5449512,136108718~1968740,147138413~7483873,169134757~5904995,175544318~4866193,185145489~6850225,192928176~8296411,218243585~5067439,225502243~4133844,237211987~13960301,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.117 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,78422427~11925226,129060064~5449512,136108718~1968740,147138413~7483873,169134757~5904995,175544318~4866193,185145489~6850225,192928176~8296411,218243585~5067439,225502243~4133844,237211987~13960301,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~7288248,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.124 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~5230633,29503754~8028782,37748736~9323039,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,77450102~12897551,128827702~5681874,136108718~1968740,147138413~7483873,169134757~5904995,175544318~4866193,184549376~16675211,218243585~11392502,237211987~13960301,251658240~11501142,264304297~6430095,273613045~1262583,295560575~7740160,306184192~9127395,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,345197613~13330662,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.124 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 14 diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,78422427~12611916,94079298~566015,126832157~7677419,136108718~1968740,147138413~7483873,169134757~11275754,185145489~16079098,218243585~11392502,237211987~13960301,253517969~879309,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~9869342,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.124 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,78422427~12611916,94079298~566015,126832157~7677419,136108718~1968740,147138413~7483873,169134757~11275754,185145489~16079098,218243585~11392502,237211987~13960301,253517969~879309,255387586~7771796,264304297~6430095,273613045~1262583,295560575~7740160,307269978~110347,308023339~9869342,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.132 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~5230633,29503754~8028782,37748736~9323039,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,126832157~7677419,136108718~1968740,147138413~7483873,169134757~11275754,184549376~16675211,218243585~11392502,237211987~13960301,251658240~11501142,264304297~6430095,273613045~1262583,295560575~7740160,306184192~11708489,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,345197613~13330662,368358100~1519512,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.132 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 15 diff [1533919~9987224,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~7677419,136108718~1968740,143476416~11145870,169134757~11275754,185145489~16079098,201287152~2371737,218243585~11392502,237211987~13960301,253517969~879309,255387586~7771796,264304297~10571331,295560575~7740160,307269978~110347,308023339~9869342,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.132 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~9987224,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~7677419,136108718~1968740,143476416~11145870,169134757~11275754,185145489~16079098,201287152~2371737,218243585~11392502,237211987~13960301,253517969~879309,255387586~7771796,264304297~10571331,295560575~7740160,307269978~110347,308023339~9869342,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.138 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~5230633,29503754~8028782,37748736~9323039,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,126832157~7677419,136108718~1968740,142606336~12015950,169134757~11275754,184549376~19109513,218243585~11392502,237211987~13960301,251658240~11501142,264304297~10571331,295560575~7740160,306184192~11708489,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,345197613~13330662,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.138 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 16 diff [838088~10683055,20029799~5892106,29503754~8028782,41106664~5965111,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~11145870,158413680~3974333,169134757~11275754,185145489~16079098,201287152~2371737,218243585~11392502,237211987~13960301,253517969~879309,255387586~7771796,264304297~10571331,295560575~7740160,307269978~110347,308023339~9869342,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.138 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,20029799~5892106,29503754~8028782,41106664~5965111,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~11145870,158413680~3974333,169134757~11275754,185145489~16079098,201287152~2371737,218243585~11392502,237211987~13960301,253517969~879309,255387586~7771796,264304297~10571331,295560575~7740160,307269978~110347,308023339~9869342,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.145 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~5892106,29503754~8028782,37748736~9323039,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,126832157~11370482,142606336~12015950,155189248~7198765,169134757~11275754,184549376~19109513,218243585~11392502,237211987~13960301,251658240~11501142,264304297~10571331,295560575~7740160,306184192~11708489,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,345197613~13330662,368358100~1519512,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.145 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 17 diff [838088~10683055,20029799~5892106,29503754~8028782,41106664~5965111,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~11392502,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~2517149,295560575~7740160,307269978~110347,308023339~9869342,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.145 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,20029799~5892106,29503754~8028782,41106664~5965111,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~11392502,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~2517149,295560575~7740160,307269978~110347,308023339~9869342,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.152 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~5892106,29503754~8028782,37748736~9323039,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~11392502,237211987~14070179,251658240~11501142,264304297~10571331,276169467~2517149,295560575~7740160,306184192~11708489,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,345197613~13330662,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.152 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 18 diff [838088~10683055,20029799~5892106,27408852~677832,29503754~8028782,41106664~5965111,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~14518264,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~9495859,295560575~7740160,307269978~110347,308023339~9869342,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.153 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,20029799~5892106,27408852~677832,29503754~8028782,41106664~5965111,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~14518264,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~9495859,295560575~7740160,307269978~110347,308023339~9869342,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.160 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~5892106,27408852~677832,29503754~8028782,37748736~9323039,48987206~726746,49928767~8999270,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~14518264,237211987~14070179,251658240~11501142,264304297~10571331,276169467~9495859,295560575~7740160,306184192~11708489,320232816~2552469,324279722~8494618,335206066~1129952,336808173~8274869,345197613~13330662,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.160 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 19 diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.160 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.167 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~15341662,237211987~14070179,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~11708489,320232816~12541524,335206066~1129952,336808173~8274869,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.167 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 8 diff [20041909~5218523,48987206~726746,225502243~4133844,255387586~7037457,295560575~7612635,335285595~1050423,350444844~7313146,375685896~3112271,389497706~7075119] 2024-09-13T00:51:21.167 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [20041909~5218523,48987206~726746,225502243~4133844,255387586~7037457,295560575~7612635,335285595~1050423,350444844~7313146,375685896~3112271,389497706~7075119] 2024-09-13T00:51:21.174 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [20041909~5218523,48987206~726746,222298112~7337975,251658240~10766803,295560575~7612635,335285595~1050423,350444844~7313146,375685896~3112271,386366614~10206211] 2024-09-13T00:51:21.174 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 9 diff [7447179~4073964,20041909~5218523,41106664~5965111,48987206~726746,147138413~922932,225502243~4133844,237941494~7012785,255387586~7037457,264304297~6430095,295560575~7612635,307269978~110347,308023339~7288248,335285595~1050423,350444844~7313146,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.175 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [7447179~4073964,20041909~5218523,41106664~5965111,48987206~726746,147138413~922932,225502243~4133844,237941494~7012785,255387586~7037457,264304297~6430095,295560575~7612635,307269978~110347,308023339~7288248,335285595~1050423,350444844~7313146,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.181 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~7326839,20041909~5218523,37748736~9323039,48987206~726746,147138413~922932,222298112~7337975,237941494~7012785,251658240~10766803,264304297~6430095,295560575~7612635,306184192~9127395,335285595~1050423,350444844~7313146,375685896~4658742,386366614~10417552] 2024-09-13T00:51:21.181 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 10 diff [1533919~394093,7447179~4073964,20041909~5218523,32783801~4748735,41106664~5965111,48987206~726746,59919134~784701,147138413~7483873,185145489~6850225,218243585~5067439,225502243~4133844,237941494~7012785,249652551~224229,255387586~7037457,264304297~6430095,295560575~7612635,307269978~110347,308023339~7288248,324279722~3309866,335285595~1050423,350444844~7313146,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.182 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20041909~5218523,32783801~4748735,41106664~5965111,48987206~726746,59919134~784701,147138413~7483873,185145489~6850225,218243585~5067439,225502243~4133844,237941494~7012785,249652551~224229,255387586~7037457,264304297~6430095,295560575~7612635,307269978~110347,308023339~7288248,324279722~3309866,335285595~1050423,350444844~7313146,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.188 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20041909~5218523,32783801~4748735,37748736~9323039,48987206~726746,59919134~784701,147138413~7483873,184549376~7446338,218243585~11392502,237941494~7012785,249652551~224229,251658240~10766803,264304297~6430095,295560575~7612635,306184192~9127395,324279722~3309866,335285595~1050423,350444844~7313146,375685896~4658742,386366614~10417552] 2024-09-13T00:51:21.188 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 11 diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,59919134~784701,64412830~6560139,78422427~5908409,147138413~7483873,169134757~5904995,185145489~6850225,195924737~5299850,218243585~5067439,225502243~4133844,237941494~7012785,245603211~5569077,255387586~7037457,264304297~6430095,273613045~1262583,295560575~7612635,307269978~110347,308023339~7288248,324279722~3309866,335206066~1129952,350444844~7313146,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.188 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,59919134~784701,64412830~6560139,78422427~5908409,147138413~7483873,169134757~5904995,185145489~6850225,195924737~5299850,218243585~5067439,225502243~4133844,237941494~7012785,245603211~5569077,255387586~7037457,264304297~6430095,273613045~1262583,295560575~7612635,307269978~110347,308023339~7288248,324279722~3309866,335206066~1129952,350444844~7313146,375685896~4658742,389497706~7286460] 2024-09-13T00:51:21.195 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~5230633,29503754~8028782,37748736~9323039,48987206~726746,59919134~784701,64412830~6560139,77450102~6880734,147138413~7483873,169134757~5904995,184549376~7446338,195924737~5299850,218243585~11392502,237941494~7012785,245603211~5569077,251658240~10766803,264304297~6430095,273613045~1262583,295560575~7612635,306184192~9127395,324279722~3309866,335206066~1129952,350444844~7313146,375685896~4658742,386366614~10417552] 2024-09-13T00:51:21.196 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 12 diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,59919134~784701,64412830~6560139,78422427~11925226,129060064~5449512,147138413~7483873,169134757~5904995,185145489~6850225,195924737~5299850,218243585~5067439,225502243~4133844,237941494~13230794,255387586~7037457,264304297~6430095,273613045~1262583,295560575~7612635,307269978~110347,308023339~7288248,324279722~8494618,335206066~1129952,336808173~4838512,348106239~9651751,375685896~4658742,384302117~3873732,389497706~7286460,406071955~8147490] 2024-09-13T00:51:21.196 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,59919134~784701,64412830~6560139,78422427~11925226,129060064~5449512,147138413~7483873,169134757~5904995,185145489~6850225,195924737~5299850,218243585~5067439,225502243~4133844,237941494~13230794,255387586~7037457,264304297~6430095,273613045~1262583,295560575~7612635,307269978~110347,308023339~7288248,324279722~8494618,335206066~1129952,336808173~4838512,348106239~9651751,375685896~4658742,384302117~3873732,389497706~7286460,406071955~8147490] 2024-09-13T00:51:21.202 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~5230633,29503754~8028782,37748736~9323039,48987206~726746,59919134~784701,64412830~6560139,77450102~12897551,128827702~5681874,147138413~7483873,169134757~5904995,184549376~7446338,195924737~5299850,218243585~11392502,237941494~13230794,251658240~10766803,264304297~6430095,273613045~1262583,295560575~7612635,306184192~9127395,324279722~8494618,335206066~1129952,336808173~4838512,345197613~12560377,375685896~4658742,384302117~12482049,406071955~8147490] 2024-09-13T00:51:21.202 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 13 diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,78422427~11925226,129060064~5449512,147138413~7483873,169134757~5904995,176889496~3521015,185145489~6850225,195924737~5299850,218243585~5067439,225502243~4133844,237941494~13230794,255387586~7037457,264304297~6430095,273613045~1262583,295560575~7612635,307269978~110347,308023339~7288248,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.202 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,78422427~11925226,129060064~5449512,147138413~7483873,169134757~5904995,176889496~3521015,185145489~6850225,195924737~5299850,218243585~5067439,225502243~4133844,237941494~13230794,255387586~7037457,264304297~6430095,273613045~1262583,295560575~7612635,307269978~110347,308023339~7288248,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.208 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~5230633,29503754~8028782,37748736~9323039,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,77450102~12897551,128827702~5681874,147138413~7483873,169134757~5904995,176889496~3521015,184549376~7446338,195924737~5299850,218243585~11392502,237941494~13230794,251658240~10766803,264304297~6430095,273613045~1262583,295560575~7612635,306184192~9127395,324279722~8494618,335206066~1129952,336808173~8274869,345197613~13330662,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.208 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 14 diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,78422427~12611916,94079298~566015,126832157~7677419,147138413~7483873,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,218243585~11392502,237941494~13230794,253517969~879309,255387586~7037457,264304297~6430095,273613045~1262583,295560575~7612635,307269978~110347,308023339~9869342,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.208 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,78422427~12611916,94079298~566015,126832157~7677419,147138413~7483873,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,218243585~11392502,237941494~13230794,253517969~879309,255387586~7037457,264304297~6430095,273613045~1262583,295560575~7612635,307269978~110347,308023339~9869342,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.215 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~5230633,29503754~8028782,37748736~9323039,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,126832157~7677419,147138413~7483873,169134757~6791266,176889496~3521015,184549376~10271010,195924737~5299850,218243585~11392502,237941494~13230794,251658240~10766803,264304297~6430095,273613045~1262583,295560575~7612635,306184192~11708489,324279722~8494618,335206066~1129952,336808173~8274869,345197613~13330662,375685896~4658742,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.215 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 15 diff [1533919~9987224,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~7677419,143476416~11145870,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~2371737,218243585~11392502,237941494~13230794,253517969~879309,255387586~7037457,264304297~10571331,295560575~7612635,307269978~110347,308023339~9869342,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.215 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~9987224,20029799~5230633,29503754~8028782,41106664~5965111,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~7677419,143476416~11145870,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~2371737,218243585~11392502,237941494~13230794,253517969~879309,255387586~7037457,264304297~10571331,295560575~7612635,307269978~110347,308023339~9869342,324279722~8494618,335206066~1129952,336808173~8274869,348106239~10422036,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.222 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~5230633,29503754~8028782,37748736~9323039,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,126832157~7677419,142606336~12015950,169134757~6791266,176889496~3521015,184549376~10271010,195924737~7734152,218243585~11392502,237941494~13230794,251658240~10766803,264304297~10571331,295560575~7612635,306184192~11708489,324279722~8494618,335206066~1129952,336808173~8274869,345197613~13330662,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.222 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 16 diff [838088~10683055,20029799~5892106,29503754~8028782,41106664~5965111,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~11145870,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~2371737,218243585~11392502,237941494~13230794,253517969~879309,255387586~7037457,264304297~10571331,295560575~7612635,307269978~110347,308023339~9869342,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.222 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,20029799~5892106,29503754~8028782,41106664~5965111,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~11145870,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~2371737,218243585~11392502,237941494~13230794,253517969~879309,255387586~7037457,264304297~10571331,295560575~7612635,307269978~110347,308023339~9869342,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.229 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~5892106,29503754~8028782,37748736~9323039,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,126832157~11370482,142606336~12015950,155189248~7198765,169134757~6791266,176889496~3521015,184549376~10271010,195924737~7734152,218243585~11392502,237941494~13230794,251658240~10766803,264304297~10571331,295560575~7612635,306184192~11708489,324279722~8494618,335206066~1129952,336808173~8274869,345197613~13330662,375685896~4658742,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.229 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 17 diff [838088~10683055,20029799~5892106,29503754~8028782,41106664~5965111,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~11392502,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~2517149,295560575~7612635,307269978~110347,308023339~9869342,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.229 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,20029799~5892106,29503754~8028782,41106664~5965111,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~11392502,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~2517149,295560575~7612635,307269978~110347,308023339~9869342,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.235 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~5892106,29503754~8028782,37748736~9323039,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~11392502,237941494~13340672,251658240~10766803,264304297~10571331,276169467~2517149,295560575~7612635,306184192~11708489,324279722~8494618,335206066~1129952,336808173~8274869,345197613~13330662,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.236 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 18 diff [838088~10683055,20029799~5892106,27408852~677832,29503754~8028782,41106664~5965111,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~14518264,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~9495859,295560575~7612635,307269978~110347,308023339~9869342,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.236 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,20029799~5892106,27408852~677832,29503754~8028782,41106664~5965111,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~14518264,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~9495859,295560575~7612635,307269978~110347,308023339~9869342,324279722~8494618,335206066~1129952,336808173~8274869,348003072~10525203,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.242 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~5892106,27408852~677832,29503754~8028782,37748736~9323039,48987206~726746,54292152~4635885,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~14518264,237941494~13340672,251658240~10766803,264304297~10571331,276169467~9495859,295560575~7612635,306184192~11708489,324279722~8494618,335206066~1129952,336808173~8274869,345197613~13330662,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.242 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 19 diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.242 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.251 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,251658240~10766803,264304297~10571331,276169467~10489226,295560575~7612635,306184192~11708489,322349784~10424556,335206066~1129952,336808173~8274869,345197613~13791043,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.251 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 9 diff [7447179~4073964,41106664~5965111,147138413~922932,237941494~7012785,264304297~6430095,307269978~110347,308023339~7288248,377731437~2613201,390858265~5925901] 2024-09-13T00:51:21.251 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [7447179~4073964,41106664~5965111,147138413~922932,237941494~7012785,264304297~6430095,307269978~110347,308023339~7288248,377731437~2613201,390858265~5925901] 2024-09-13T00:51:21.258 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~7326839,37748736~9323039,147138413~922932,237941494~7012785,264304297~6430095,306184192~9127395,377731437~2613201,390858265~5925901] 2024-09-13T00:51:21.258 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 10 diff [1533919~394093,7447179~4073964,32783801~4748735,41106664~5965111,59919134~784701,147138413~7483873,185145489~6850225,218243585~5067439,237941494~7012785,249652551~224229,264304297~6430095,307269978~110347,308023339~7288248,324279722~3309866,377731437~2613201,390858265~5925901] 2024-09-13T00:51:21.258 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,32783801~4748735,41106664~5965111,59919134~784701,147138413~7483873,185145489~6850225,218243585~5067439,237941494~7012785,249652551~224229,264304297~6430095,307269978~110347,308023339~7288248,324279722~3309866,377731437~2613201,390858265~5925901] 2024-09-13T00:51:21.265 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,32783801~4748735,37748736~9323039,59919134~784701,147138413~7483873,184549376~7446338,218243585~5067439,237941494~7012785,249652551~224229,264304297~6430095,306184192~9127395,324279722~3309866,377731437~2613201,390858265~5925901] 2024-09-13T00:51:21.265 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 11 diff [1533919~394093,7447179~4073964,20029799~2994873,29503754~8028782,41106664~5965111,59919134~784701,64412830~6560139,78422427~5908409,147138413~7483873,169134757~5904995,185145489~6850225,195924737~5299850,218243585~5067439,237941494~7012785,245603211~5569077,264304297~6430095,273613045~1262583,307269978~110347,308023339~7288248,324279722~3309866,335206066~179599,377731437~2613201,390858265~5925901] 2024-09-13T00:51:21.265 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~2994873,29503754~8028782,41106664~5965111,59919134~784701,64412830~6560139,78422427~5908409,147138413~7483873,169134757~5904995,185145489~6850225,195924737~5299850,218243585~5067439,237941494~7012785,245603211~5569077,264304297~6430095,273613045~1262583,307269978~110347,308023339~7288248,324279722~3309866,335206066~179599,377731437~2613201,390858265~5925901] 2024-09-13T00:51:21.271 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~2994873,29503754~8028782,37748736~9323039,59919134~784701,64412830~6560139,77450102~6880734,147138413~7483873,169134757~5904995,184549376~7446338,195924737~5299850,218243585~5067439,237941494~7012785,245603211~5569077,264304297~6430095,273613045~1262583,306184192~9127395,324279722~3309866,335206066~179599,377731437~2613201,390858265~5925901] 2024-09-13T00:51:21.271 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 12 diff [1533919~394093,7447179~4073964,20029799~2994873,29503754~8028782,41106664~5965111,59919134~784701,64412830~6560139,78422427~11925226,129060064~5449512,147138413~7483873,169134757~5904995,185145489~6850225,195924737~5299850,218243585~5067439,237941494~13230794,264304297~6430095,273613045~1262583,307269978~110347,308023339~7288248,324279722~8494618,335206066~179599,336808173~4838512,348106239~3988747,377731437~2613201,384302117~3873732,390858265~5925901,406071955~8147490] 2024-09-13T00:51:21.271 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~2994873,29503754~8028782,41106664~5965111,59919134~784701,64412830~6560139,78422427~11925226,129060064~5449512,147138413~7483873,169134757~5904995,185145489~6850225,195924737~5299850,218243585~5067439,237941494~13230794,264304297~6430095,273613045~1262583,307269978~110347,308023339~7288248,324279722~8494618,335206066~179599,336808173~4838512,348106239~3988747,377731437~2613201,384302117~3873732,390858265~5925901,406071955~8147490] 2024-09-13T00:51:21.277 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~2994873,29503754~8028782,37748736~9323039,59919134~784701,64412830~6560139,77450102~12897551,128827702~5681874,147138413~7483873,169134757~5904995,184549376~7446338,195924737~5299850,218243585~5067439,237941494~13230794,264304297~6430095,273613045~1262583,306184192~9127395,324279722~8494618,335206066~179599,336808173~4838512,345197613~6897373,377731437~2613201,384302117~3873732,390858265~5925901,406071955~8147490] 2024-09-13T00:51:21.278 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 13 diff [1533919~394093,7447179~4073964,20029799~2994873,29503754~8028782,41106664~5965111,54292152~4635885,59919134~784701,64412830~6560139,78422427~11925226,129060064~5449512,147138413~7483873,169134757~5904995,176889496~3521015,185145489~6850225,195924737~5299850,218243585~5067439,237941494~13230794,264304297~6430095,273613045~1262583,297168147~5073715,307269978~110347,308023339~7288248,324279722~8494618,335206066~179599,336808173~8274869,348106239~10422036,377731437~2613201,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.278 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~2994873,29503754~8028782,41106664~5965111,54292152~4635885,59919134~784701,64412830~6560139,78422427~11925226,129060064~5449512,147138413~7483873,169134757~5904995,176889496~3521015,185145489~6850225,195924737~5299850,218243585~5067439,237941494~13230794,264304297~6430095,273613045~1262583,297168147~5073715,307269978~110347,308023339~7288248,324279722~8494618,335206066~179599,336808173~8274869,348106239~10422036,377731437~2613201,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.284 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~2994873,29503754~8028782,37748736~9323039,54292152~4635885,59919134~784701,64412830~6560139,77450102~12897551,128827702~5681874,147138413~7483873,169134757~5904995,176889496~3521015,184549376~7446338,195924737~5299850,218243585~5067439,237941494~13230794,264304297~6430095,273613045~1262583,297168147~5073715,306184192~9127395,324279722~8494618,335206066~179599,336808173~8274869,345197613~13330662,377731437~2613201,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.284 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 14 diff [1533919~394093,7447179~4073964,20029799~2994873,29503754~8028782,41106664~5965111,54292152~4635885,59919134~784701,64412830~6560139,78422427~12611916,94079298~566015,126832157~7677419,147138413~7483873,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,218243585~9926339,237941494~13230794,253517969~879309,264304297~6430095,273613045~1262583,297168147~5073715,307269978~110347,308023339~9869342,324279722~8494618,335206066~179599,336808173~8274869,348106239~10422036,377731437~2613201,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.284 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7447179~4073964,20029799~2994873,29503754~8028782,41106664~5965111,54292152~4635885,59919134~784701,64412830~6560139,78422427~12611916,94079298~566015,126832157~7677419,147138413~7483873,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,218243585~9926339,237941494~13230794,253517969~879309,264304297~6430095,273613045~1262583,297168147~5073715,307269978~110347,308023339~9869342,324279722~8494618,335206066~179599,336808173~8274869,348106239~10422036,377731437~2613201,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.290 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,4194304~7326839,20029799~2994873,29503754~8028782,37748736~9323039,54292152~4635885,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,126832157~7677419,147138413~7483873,169134757~6791266,176889496~3521015,184549376~10271010,195924737~5299850,218243585~9926339,237941494~13230794,253517969~879309,264304297~6430095,273613045~1262583,297168147~5073715,306184192~11708489,324279722~8494618,335206066~179599,336808173~8274869,345197613~13330662,377731437~2613201,381207933~15576233,406071955~8147490] 2024-09-13T00:51:21.290 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 15 diff [1533919~9987224,20029799~2994873,29503754~8028782,41106664~5965111,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~7677419,143476416~11145870,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~2371737,218243585~9926339,237941494~13230794,253517969~879309,256862597~5111784,264304297~10571331,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,324279722~8494618,335206066~179599,336808173~8274869,348106239~10422036,377731437~2613201,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.290 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~9987224,20029799~2994873,29503754~8028782,41106664~5965111,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~7677419,143476416~11145870,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~2371737,218243585~9926339,237941494~13230794,253517969~879309,256862597~5111784,264304297~10571331,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,324279722~8494618,335206066~179599,336808173~8274869,348106239~10422036,377731437~2613201,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.297 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~2994873,29503754~8028782,37748736~9323039,54292152~4635885,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,126832157~7677419,142606336~12015950,169134757~6791266,176889496~3521015,184549376~10271010,195924737~7734152,218243585~9926339,237941494~13230794,253517969~879309,256862597~5111784,264304297~10571331,297168147~5073715,302702098~43268,306184192~11708489,324279722~8494618,335206066~179599,336808173~8274869,345197613~13330662,377731437~2613201,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.297 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 16 diff [838088~10683055,20029799~2994873,25074977~846928,29503754~8028782,41106664~5965111,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~11145870,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~2371737,218243585~10156924,237941494~13230794,253517969~879309,256862597~5111784,264304297~10571331,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,324279722~8494618,335206066~179599,336808173~8274869,348003072~10525203,377731437~2613201,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.297 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,20029799~2994873,25074977~846928,29503754~8028782,41106664~5965111,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~11145870,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~2371737,218243585~10156924,237941494~13230794,253517969~879309,256862597~5111784,264304297~10571331,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,324279722~8494618,335206066~179599,336808173~8274869,348003072~10525203,377731437~2613201,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.304 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~2994873,25074977~846928,29503754~8028782,37748736~9323039,54292152~4635885,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,126832157~11370482,142606336~12015950,155189248~7198765,169134757~6791266,176889496~3521015,184549376~10271010,195924737~7734152,218243585~10156924,237941494~13230794,253517969~879309,256862597~5111784,264304297~10571331,297168147~5073715,302702098~43268,306184192~11708489,324279722~8494618,335206066~179599,336808173~8274869,345197613~13330662,377731437~2613201,381207933~15576233,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.304 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 17 diff [838088~10683055,20029799~2994873,25074977~846928,29503754~8028782,41106664~5965111,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~10156924,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~2517149,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,324279722~8494618,335206066~179599,336808173~8274869,348003072~10525203,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.304 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,20029799~2994873,25074977~846928,29503754~8028782,41106664~5965111,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~10156924,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~2517149,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,324279722~8494618,335206066~179599,336808173~8274869,348003072~10525203,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.311 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~2994873,25074977~846928,29503754~8028782,37748736~9323039,54292152~4635885,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~10156924,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~2517149,297168147~5073715,302702098~43268,306184192~11708489,324279722~8494618,335206066~179599,336808173~8274869,345197613~13330662,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.311 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 18 diff [838088~10683055,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~5965111,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~14518264,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~9495859,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,324279722~8494618,335206066~179599,336808173~8274869,348003072~10525203,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.311 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~5965111,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~14518264,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~9495859,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,324279722~8494618,335206066~179599,336808173~8274869,348003072~10525203,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.316 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,37748736~9323039,54292152~4635885,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~14518264,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~9495859,297168147~5073715,302702098~43268,306184192~11708489,324279722~8494618,335206066~179599,336808173~8274869,345197613~13330662,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.317 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 19 diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.317 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.323 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,37748736~10633085,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,306184192~11708489,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.323 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 10 diff [1533919~394093,7753840~1657202,32783801~4748735,59919134~784701,147811050~6811236,185145489~6850225,218243585~5067439,249652551~224229,324279722~3309866] 2024-09-13T00:51:21.323 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7753840~1657202,32783801~4748735,59919134~784701,147811050~6811236,185145489~6850225,218243585~5067439,249652551~224229,324279722~3309866] 2024-09-13T00:51:21.330 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,7753840~1657202,32783801~4748735,59919134~784701,147811050~6811236,184549376~7446338,218243585~5067439,249652551~224229,324279722~3309866] 2024-09-13T00:51:21.330 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 11 diff [1533919~394093,7753840~1657202,20029799~2994873,29503754~8028782,59919134~784701,64412830~6560139,78422427~5908409,147811050~6811236,169134757~5904995,185145489~6850225,195924737~5299850,218243585~5067439,245603211~5569077,273613045~1262583,324279722~3309866,335206066~179599] 2024-09-13T00:51:21.330 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7753840~1657202,20029799~2994873,29503754~8028782,59919134~784701,64412830~6560139,78422427~5908409,147811050~6811236,169134757~5904995,185145489~6850225,195924737~5299850,218243585~5067439,245603211~5569077,273613045~1262583,324279722~3309866,335206066~179599] 2024-09-13T00:51:21.336 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,7753840~1657202,20029799~2994873,29503754~8028782,59919134~784701,64412830~6560139,77450102~6880734,147811050~6811236,169134757~5904995,184549376~7446338,195924737~5299850,218243585~5067439,245603211~5569077,273613045~1262583,324279722~3309866,335206066~179599] 2024-09-13T00:51:21.336 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 12 diff [1533919~394093,7753840~1657202,20029799~2994873,29503754~8028782,59919134~784701,64412830~6560139,78422427~11925226,129060064~5449512,147811050~6811236,169134757~5904995,185145489~6850225,195924737~5299850,218243585~5067439,241666235~9506053,273613045~1262583,324279722~8494618,335206066~179599,336808173~4838512,348106239~3988747,384302117~3873732,406071955~8147490] 2024-09-13T00:51:21.336 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7753840~1657202,20029799~2994873,29503754~8028782,59919134~784701,64412830~6560139,78422427~11925226,129060064~5449512,147811050~6811236,169134757~5904995,185145489~6850225,195924737~5299850,218243585~5067439,241666235~9506053,273613045~1262583,324279722~8494618,335206066~179599,336808173~4838512,348106239~3988747,384302117~3873732,406071955~8147490] 2024-09-13T00:51:21.343 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,7753840~1657202,20029799~2994873,29503754~8028782,59919134~784701,64412830~6560139,77450102~12897551,128827702~5681874,147811050~6811236,169134757~5904995,184549376~7446338,195924737~5299850,218243585~5067439,241666235~9506053,273613045~1262583,324279722~8494618,335206066~179599,336808173~4838512,345197613~6897373,384302117~3873732,406071955~8147490] 2024-09-13T00:51:21.343 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 13 diff [1533919~394093,7753840~1657202,20029799~2994873,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,78422427~11925226,129060064~5449512,147811050~6811236,169134757~5904995,176889496~3521015,185145489~6850225,195924737~5299850,218243585~5067439,241666235~9506053,264933973~599451,273613045~1262583,297168147~5073715,324279722~8494618,335206066~179599,336808173~8274869,348106239~10422036,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.343 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7753840~1657202,20029799~2994873,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,78422427~11925226,129060064~5449512,147811050~6811236,169134757~5904995,176889496~3521015,185145489~6850225,195924737~5299850,218243585~5067439,241666235~9506053,264933973~599451,273613045~1262583,297168147~5073715,324279722~8494618,335206066~179599,336808173~8274869,348106239~10422036,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.349 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,7753840~1657202,20029799~2994873,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,77450102~12897551,128827702~5681874,147811050~6811236,169134757~5904995,176889496~3521015,184549376~7446338,195924737~5299850,218243585~5067439,241666235~9506053,264933973~599451,273613045~1262583,297168147~5073715,324279722~8494618,335206066~179599,336808173~8274869,345197613~13330662,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.349 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 14 diff [1533919~394093,7753840~1657202,20029799~2994873,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,78422427~12611916,94079298~566015,126832157~7677419,147811050~6811236,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,218243585~9926339,241666235~9506053,253517969~879309,264933973~599451,273613045~1262583,297168147~5073715,310505352~7387329,324279722~8494618,335206066~179599,336808173~8274869,348106239~10422036,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.349 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~394093,7753840~1657202,20029799~2994873,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,78422427~12611916,94079298~566015,126832157~7677419,147811050~6811236,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,218243585~9926339,241666235~9506053,253517969~879309,264933973~599451,273613045~1262583,297168147~5073715,310505352~7387329,324279722~8494618,335206066~179599,336808173~8274869,348106239~10422036,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.355 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~1928012,7753840~1657202,20029799~2994873,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,126832157~7677419,147811050~6811236,169134757~6791266,176889496~3521015,184549376~10271010,195924737~5299850,218243585~9926339,241666235~9506053,253517969~879309,264933973~599451,273613045~1262583,297168147~5073715,310505352~7387329,324279722~8494618,335206066~179599,336808173~8274869,345197613~13330662,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.355 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 15 diff [1533919~7877123,20029799~2994873,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~7677419,143476416~11145870,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~2371737,218243585~9926339,241666235~9506053,253517969~879309,256862597~5111784,264933973~599451,270407338~4468290,297168147~5073715,302702098~43268,310505352~7387329,324279722~8494618,335206066~179599,336808173~8274869,348106239~10422036,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.355 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1533919~7877123,20029799~2994873,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~7677419,143476416~11145870,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~2371737,218243585~9926339,241666235~9506053,253517969~879309,256862597~5111784,264933973~599451,270407338~4468290,297168147~5073715,302702098~43268,310505352~7387329,324279722~8494618,335206066~179599,336808173~8274869,348106239~10422036,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.361 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~9411042,20029799~2994873,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,126832157~7677419,142606336~12015950,169134757~6791266,176889496~3521015,184549376~10271010,195924737~7734152,218243585~9926339,241666235~9506053,253517969~879309,256862597~5111784,264933973~599451,270407338~4468290,297168147~5073715,302702098~43268,310505352~7387329,324279722~8494618,335206066~179599,336808173~8274869,345197613~13330662,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.361 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 16 diff [838088~8572954,20029799~2994873,25074977~846928,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~11145870,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~2371737,218243585~10156924,241666235~9506053,253517969~879309,256862597~5111784,264933973~599451,270407338~4468290,297168147~5073715,302702098~43268,310505352~7387329,324279722~8494618,335206066~179599,336808173~8274869,348003072~10525203,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.362 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~8572954,20029799~2994873,25074977~846928,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~11145870,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~2371737,218243585~10156924,241666235~9506053,253517969~879309,256862597~5111784,264933973~599451,270407338~4468290,297168147~5073715,302702098~43268,310505352~7387329,324279722~8494618,335206066~179599,336808173~8274869,348003072~10525203,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.368 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~9411042,20029799~2994873,25074977~846928,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,126832157~11370482,142606336~12015950,155189248~7198765,169134757~6791266,176889496~3521015,184549376~10271010,195924737~7734152,218243585~10156924,241666235~9506053,253517969~879309,256862597~5111784,264933973~599451,270407338~4468290,297168147~5073715,302702098~43268,310505352~7387329,324279722~8494618,335206066~179599,336808173~8274869,345197613~13330662,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.368 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 17 diff [838088~8572954,20029799~2994873,25074977~846928,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~10156924,241666235~9615931,253517969~879309,256862597~5111784,264793518~2809034,270407338~4468290,276169467~2517149,297168147~5073715,302702098~43268,310505352~7387329,324279722~8494618,335206066~179599,336808173~8274869,348003072~10525203,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.368 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~8572954,20029799~2994873,25074977~846928,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~10156924,241666235~9615931,253517969~879309,256862597~5111784,264793518~2809034,270407338~4468290,276169467~2517149,297168147~5073715,302702098~43268,310505352~7387329,324279722~8494618,335206066~179599,336808173~8274869,348003072~10525203,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.374 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~9411042,20029799~2994873,25074977~846928,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~10156924,241666235~9615931,253517969~879309,256862597~5111784,264793518~2809034,270407338~4468290,276169467~2517149,297168147~5073715,302702098~43268,310505352~7387329,324279722~8494618,335206066~179599,336808173~8274869,345197613~13330662,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.374 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 18 diff [838088~8572954,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~14518264,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~9495859,297168147~5073715,302702098~43268,310505352~7387329,324279722~8494618,335206066~179599,336808173~8274869,348003072~10525203,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.374 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~8572954,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~14518264,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~9495859,297168147~5073715,302702098~43268,310505352~7387329,324279722~8494618,335206066~179599,336808173~8274869,348003072~10525203,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.380 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~9411042,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,54292152~4635885,59919134~784701,64412830~6560139,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~14518264,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~9495859,297168147~5073715,302702098~43268,310505352~7387329,324279722~8494618,335206066~179599,336808173~8274869,345197613~13330662,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.380 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 19 diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.380 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.387 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~9411042,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.387 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 11 diff [20029799~2994873,29503754~5993847,64412830~6560139,78422427~5908409,169134757~5904995,195924737~5299850,245603211~5569077,273613045~1262583,335206066~179599] 2024-09-13T00:51:21.387 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [20029799~2994873,29503754~5993847,64412830~6560139,78422427~5908409,169134757~5904995,195924737~5299850,245603211~5569077,273613045~1262583,335206066~179599] 2024-09-13T00:51:21.393 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [20029799~2994873,29503754~5993847,64412830~6560139,77450102~6880734,169134757~5904995,195924737~5299850,245603211~5569077,273613045~1262583,335206066~179599] 2024-09-13T00:51:21.393 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 12 diff [20029799~2994873,29503754~5993847,64412830~6560139,78422427~11925226,129060064~5449512,169134757~5904995,195924737~5299850,241666235~9506053,273613045~1262583,326698751~6075589,335206066~179599,336808173~4838512,348106239~3988747,384302117~3873732,406071955~8147490] 2024-09-13T00:51:21.394 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [20029799~2994873,29503754~5993847,64412830~6560139,78422427~11925226,129060064~5449512,169134757~5904995,195924737~5299850,241666235~9506053,273613045~1262583,326698751~6075589,335206066~179599,336808173~4838512,348106239~3988747,384302117~3873732,406071955~8147490] 2024-09-13T00:51:21.399 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [20029799~2994873,29503754~5993847,64412830~6560139,77450102~12897551,128827702~5681874,169134757~5904995,195924737~5299850,241666235~9506053,273613045~1262583,326698751~6075589,335206066~179599,336808173~4838512,345197613~6897373,384302117~3873732,406071955~8147490] 2024-09-13T00:51:21.399 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 13 diff [20029799~2994873,29503754~5993847,54292152~4635885,64412830~6560139,78422427~11925226,129060064~5449512,169134757~5904995,176889496~3521015,195924737~5299850,241666235~9506053,264933973~599451,273613045~1262583,297168147~5073715,326698751~6075589,335206066~179599,336808173~8274869,348106239~10422036,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.400 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [20029799~2994873,29503754~5993847,54292152~4635885,64412830~6560139,78422427~11925226,129060064~5449512,169134757~5904995,176889496~3521015,195924737~5299850,241666235~9506053,264933973~599451,273613045~1262583,297168147~5073715,326698751~6075589,335206066~179599,336808173~8274869,348106239~10422036,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.406 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [20029799~2994873,29503754~5993847,54292152~4635885,64412830~6560139,77450102~12897551,128827702~5681874,169134757~5904995,176889496~3521015,195924737~5299850,241666235~9506053,264933973~599451,273613045~1262583,297168147~5073715,326698751~6075589,335206066~179599,336808173~8274869,345197613~13330662,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.406 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 14 diff [20029799~2994873,29503754~5993847,54292152~4635885,64412830~6560139,78422427~12611916,94079298~566015,126832157~7677419,169134757~6791266,176889496~3521015,189320558~5499828,195924737~5299850,221082790~7087134,241666235~9506053,253517969~879309,264933973~599451,273613045~1262583,297168147~5073715,310505352~7387329,326698751~6075589,335206066~179599,336808173~8274869,348106239~10422036,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.406 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [20029799~2994873,29503754~5993847,54292152~4635885,64412830~6560139,78422427~12611916,94079298~566015,126832157~7677419,169134757~6791266,176889496~3521015,189320558~5499828,195924737~5299850,221082790~7087134,241666235~9506053,253517969~879309,264933973~599451,273613045~1262583,297168147~5073715,310505352~7387329,326698751~6075589,335206066~179599,336808173~8274869,348106239~10422036,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.413 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [20029799~2994873,29503754~5993847,54292152~4635885,64412830~6560139,77450102~13584241,94079298~566015,126832157~7677419,169134757~6791266,176889496~3521015,189320558~5499828,195924737~5299850,221082790~7087134,241666235~9506053,253517969~879309,264933973~599451,273613045~1262583,297168147~5073715,310505352~7387329,326698751~6075589,335206066~179599,336808173~8274869,345197613~13330662,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.413 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 15 diff [1657775~7078413,20029799~2994873,29503754~5993847,54292152~4635885,64412830~6560139,77484618~13549725,94079298~566015,126832157~7677419,143476416~7974968,169134757~6791266,176889496~3521015,189320558~5499828,195924737~5299850,201287152~2371737,221082790~7087134,241666235~9506053,253517969~879309,256862597~5111784,264933973~599451,270407338~4468290,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,335206066~179599,336808173~8274869,348106239~10422036,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.413 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1657775~7078413,20029799~2994873,29503754~5993847,54292152~4635885,64412830~6560139,77484618~13549725,94079298~566015,126832157~7677419,143476416~7974968,169134757~6791266,176889496~3521015,189320558~5499828,195924737~5299850,201287152~2371737,221082790~7087134,241666235~9506053,253517969~879309,256862597~5111784,264933973~599451,270407338~4468290,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,335206066~179599,336808173~8274869,348106239~10422036,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.418 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [1657775~7078413,20029799~2994873,29503754~5993847,54292152~4635885,64412830~6560139,77450102~13584241,94079298~566015,126832157~7677419,142606336~8845048,169134757~6791266,176889496~3521015,189320558~5499828,195924737~7734152,221082790~7087134,241666235~9506053,253517969~879309,256862597~5111784,264933973~599451,270407338~4468290,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,335206066~179599,336808173~8274869,345197613~13330662,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.418 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 16 diff [838088~7898100,20029799~2994873,25074977~846928,29503754~5993847,54292152~4635885,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~7974968,158413680~3974333,169134757~6791266,176889496~3521015,185779876~9040510,195924737~5299850,201287152~2371737,221082790~7317719,241666235~9506053,253517969~879309,256862597~5111784,264933973~599451,270407338~4468290,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,335206066~179599,336808173~8274869,348003072~10525203,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.419 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,20029799~2994873,25074977~846928,29503754~5993847,54292152~4635885,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~7974968,158413680~3974333,169134757~6791266,176889496~3521015,185779876~9040510,195924737~5299850,201287152~2371737,221082790~7317719,241666235~9506053,253517969~879309,256862597~5111784,264933973~599451,270407338~4468290,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,335206066~179599,336808173~8274869,348003072~10525203,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.425 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,20029799~2994873,25074977~846928,29503754~5993847,54292152~4635885,64412830~6560139,77450102~13584241,94079298~566015,126832157~11370482,142606336~8845048,155189248~7198765,169134757~6791266,176889496~3521015,185779876~9040510,195924737~7734152,221082790~7317719,241666235~9506053,253517969~879309,256862597~5111784,264933973~599451,270407338~4468290,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,335206066~179599,336808173~8274869,345197613~13330662,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.425 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 17 diff [838088~7898100,20029799~2994873,25074977~846928,29503754~5993847,54292152~4635885,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~7317719,241666235~9615931,253517969~879309,256862597~5111784,264793518~2809034,270407338~4468290,276169467~2517149,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,335206066~179599,336808173~8274869,348003072~10525203,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.425 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,20029799~2994873,25074977~846928,29503754~5993847,54292152~4635885,64412830~6560139,77484618~13549725,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~7317719,241666235~9615931,253517969~879309,256862597~5111784,264793518~2809034,270407338~4468290,276169467~2517149,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,335206066~179599,336808173~8274869,348003072~10525203,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.431 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,20029799~2994873,25074977~846928,29503754~5993847,54292152~4635885,64412830~6560139,77450102~13584241,94079298~566015,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,185471528~9348858,195924737~21572614,221082790~7317719,241666235~9615931,253517969~879309,256862597~5111784,264793518~2809034,270407338~4468290,276169467~2517149,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,335206066~179599,336808173~8274869,345197613~13330662,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.431 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 18 diff [838088~7898100,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,54292152~4635885,64412830~6560139,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~11679059,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~9495859,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,335206066~179599,336808173~8274869,348003072~10525203,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.431 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,54292152~4635885,64412830~6560139,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~11679059,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~9495859,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,335206066~179599,336808173~8274869,348003072~10525203,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.438 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,54292152~4635885,64412830~6560139,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,185471528~9348858,195924737~21572614,221082790~11679059,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~9495859,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,335206066~179599,336808173~8274869,345197613~13330662,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.438 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 19 diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.438 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.445 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,185471528~9348858,195924737~21572614,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.445 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to 12 diff [32229647~1744274,82234999~8112654,129060064~5449512,241666235~4926956,326698751~6075589,336808173~4838512,348106239~3988747,384302117~3873732,406071955~8147490] 2024-09-13T00:51:21.445 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [32229647~1744274,82234999~8112654,129060064~5449512,241666235~4926956,326698751~6075589,336808173~4838512,348106239~3988747,384302117~3873732,406071955~8147490] 2024-09-13T00:51:21.451 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [32229647~1744274,82234999~8112654,128827702~5681874,241666235~4926956,326698751~6075589,336808173~4838512,345197613~6897373,384302117~3873732,406071955~8147490] 2024-09-13T00:51:21.451 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to 13 diff [32229647~1744274,54292152~4635885,82234999~8112654,129060064~5449512,170779539~568367,176889496~3521015,241666235~4926956,264933973~599451,297168147~5073715,326698751~6075589,336808173~8274869,348106239~10422036,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.451 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [32229647~1744274,54292152~4635885,82234999~8112654,129060064~5449512,170779539~568367,176889496~3521015,241666235~4926956,264933973~599451,297168147~5073715,326698751~6075589,336808173~8274869,348106239~10422036,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.457 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [32229647~1744274,54292152~4635885,82234999~8112654,128827702~5681874,170779539~568367,176889496~3521015,241666235~4926956,264933973~599451,297168147~5073715,326698751~6075589,336808173~8274869,345197613~13330662,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.457 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to 14 diff [32229647~1744274,54292152~4635885,81835783~9198560,94079298~566015,126832157~7677419,170779539~568367,171824257~4101766,176889496~3521015,189320558~5499828,221082790~7087134,241666235~4926956,253517969~879309,264933973~599451,297168147~5073715,310505352~7387329,326698751~6075589,336808173~8274869,348106239~10422036,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.457 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [32229647~1744274,54292152~4635885,81835783~9198560,94079298~566015,126832157~7677419,170779539~568367,171824257~4101766,176889496~3521015,189320558~5499828,221082790~7087134,241666235~4926956,253517969~879309,264933973~599451,297168147~5073715,310505352~7387329,326698751~6075589,336808173~8274869,348106239~10422036,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.463 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [32229647~1744274,54292152~4635885,81835783~9198560,94079298~566015,126832157~7677419,170779539~568367,171824257~4101766,176889496~3521015,189320558~5499828,221082790~7087134,241666235~4926956,253517969~879309,264933973~599451,297168147~5073715,310505352~7387329,326698751~6075589,336808173~8274869,345197613~13330662,381207933~12373750,406071955~8147490] 2024-09-13T00:51:21.463 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to 15 diff [1657775~7078413,32229647~1744274,54292152~4635885,77484618~13549725,94079298~566015,126832157~7677419,143476416~7974968,170779539~568367,171824257~4101766,176889496~3521015,189320558~5499828,201287152~2371737,221082790~7087134,241666235~4926956,253517969~879309,256862597~5111784,264933973~599451,270407338~3576644,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,336808173~8274869,348106239~10422036,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.464 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1657775~7078413,32229647~1744274,54292152~4635885,77484618~13549725,94079298~566015,126832157~7677419,143476416~7974968,170779539~568367,171824257~4101766,176889496~3521015,189320558~5499828,201287152~2371737,221082790~7087134,241666235~4926956,253517969~879309,256862597~5111784,264933973~599451,270407338~3576644,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,336808173~8274869,348106239~10422036,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.470 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [1657775~7078413,32229647~1744274,54292152~4635885,77484618~13549725,94079298~566015,126832157~7677419,142606336~8845048,170779539~568367,171824257~4101766,176889496~3521015,189320558~5499828,201224587~2434302,221082790~7087134,241666235~4926956,253517969~879309,256862597~5111784,264933973~599451,270407338~3576644,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,336808173~8274869,345197613~13330662,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.470 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to 16 diff [838088~7898100,25074977~846928,32229647~1744274,54292152~4635885,77484618~13549725,94079298~566015,126832157~11370482,143476416~7974968,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185779876~9040510,201287152~2371737,221082790~7317719,241666235~4926956,247799016~2251538,253517969~879309,256862597~5111784,264933973~599451,270407338~3576644,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,336808173~8274869,348003072~10525203,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.470 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,25074977~846928,32229647~1744274,54292152~4635885,77484618~13549725,94079298~566015,126832157~11370482,143476416~7974968,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185779876~9040510,201287152~2371737,221082790~7317719,241666235~4926956,247799016~2251538,253517969~879309,256862597~5111784,264933973~599451,270407338~3576644,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,336808173~8274869,348003072~10525203,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.477 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,25074977~846928,32229647~1744274,54292152~4635885,77484618~13549725,94079298~566015,126832157~11370482,142606336~8845048,155189248~7198765,170779539~568367,171824257~4101766,176889496~3521015,185779876~9040510,201224587~2434302,221082790~7317719,241666235~4926956,247799016~2251538,253517969~879309,256862597~5111784,264933973~599451,270407338~3576644,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,336808173~8274869,345197613~13330662,381207933~12373750,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.477 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to 17 diff [838088~7898100,25074977~846928,32229647~1744274,54292152~4635885,77484618~13549725,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~7317719,241666235~9615931,253517969~879309,256862597~5111784,264793518~2809034,270407338~3576644,276169467~2517149,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,336808173~8274869,348003072~10525203,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.477 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,25074977~846928,32229647~1744274,54292152~4635885,77484618~13549725,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~7317719,241666235~9615931,253517969~879309,256862597~5111784,264793518~2809034,270407338~3576644,276169467~2517149,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,336808173~8274869,348003072~10525203,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.483 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,25074977~846928,32229647~1744274,54292152~4635885,77484618~13549725,94079298~566015,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~7317719,241666235~9615931,253517969~879309,256862597~5111784,264793518~2809034,270407338~3576644,276169467~2517149,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,336808173~8274869,345197613~13330662,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.483 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to 18 diff [838088~7898100,25074977~846928,27408852~677832,32229647~1744274,54292152~4635885,65799589~3651744,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~11679059,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~9495859,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,336808173~8274869,348003072~10525203,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.484 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,25074977~846928,27408852~677832,32229647~1744274,54292152~4635885,65799589~3651744,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~11679059,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~9495859,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,336808173~8274869,348003072~10525203,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.490 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,25074977~846928,27408852~677832,32229647~1744274,54292152~4635885,65799589~3651744,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~11679059,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~9495859,297168147~5073715,302702098~43268,310505352~7387329,326427340~6347000,336808173~8274869,345197613~13330662,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.490 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to 19 diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.490 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.496 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:21.497 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to 13 diff [54292152~4635885,89298043~942348,170779539~568367,176889496~3521015,264933973~599451,297168147~5073715,340517251~4565791,351050230~7478045,381207933~6058720,387676088~5905595] 2024-09-13T00:51:21.497 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [54292152~4635885,89298043~942348,170779539~568367,176889496~3521015,264933973~599451,297168147~5073715,340517251~4565791,351050230~7478045,381207933~6058720,387676088~5905595] 2024-09-13T00:51:21.502 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [54292152~4635885,89298043~942348,170779539~568367,176889496~3521015,264933973~599451,297168147~5073715,340517251~4565791,351050230~7478045,381207933~6058720,387676088~5905595] 2024-09-13T00:51:21.502 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to 14 diff [54292152~4635885,81835783~4369273,86388736~4645607,94079298~566015,126832157~6964267,170779539~568367,171824257~4101766,176889496~3521015,189320558~5499828,221082790~7087134,253517969~879309,264933973~599451,297168147~5073715,310505352~7387329,340517251~4565791,350426872~8101403,381207933~6058720,387676088~5905595] 2024-09-13T00:51:21.502 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [54292152~4635885,81835783~4369273,86388736~4645607,94079298~566015,126832157~6964267,170779539~568367,171824257~4101766,176889496~3521015,189320558~5499828,221082790~7087134,253517969~879309,264933973~599451,297168147~5073715,310505352~7387329,340517251~4565791,350426872~8101403,381207933~6058720,387676088~5905595] 2024-09-13T00:51:21.508 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [54292152~4635885,81835783~4369273,86388736~4645607,94079298~566015,126832157~6964267,170779539~568367,171824257~4101766,176889496~3521015,189320558~5499828,221082790~7087134,253517969~879309,264933973~599451,297168147~5073715,310505352~7387329,340517251~4565791,350426872~8101403,381207933~6058720,387676088~5905595] 2024-09-13T00:51:21.509 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to 15 diff [1657775~7078413,54292152~4635885,77484618~8720438,86388736~4645607,94079298~566015,126832157~6964267,143476416~7974968,170779539~568367,171824257~4101766,176889496~3521015,189320558~5499828,201287152~2371737,221082790~7087134,253517969~879309,256862597~5111784,264933973~599451,270407338~3576644,297168147~5073715,302702098~43268,310505352~7387329,326427340~4602689,340517251~4565791,350426872~8101403,381207933~6058720,387676088~5905595,399030489~1461454] 2024-09-13T00:51:21.509 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1657775~7078413,54292152~4635885,77484618~8720438,86388736~4645607,94079298~566015,126832157~6964267,143476416~7974968,170779539~568367,171824257~4101766,176889496~3521015,189320558~5499828,201287152~2371737,221082790~7087134,253517969~879309,256862597~5111784,264933973~599451,270407338~3576644,297168147~5073715,302702098~43268,310505352~7387329,326427340~4602689,340517251~4565791,350426872~8101403,381207933~6058720,387676088~5905595,399030489~1461454] 2024-09-13T00:51:21.515 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [1657775~7078413,54292152~4635885,77484618~8720438,86388736~4645607,94079298~566015,126832157~6964267,142606336~8845048,170779539~568367,171824257~4101766,176889496~3521015,189320558~5499828,201224587~2434302,221082790~7087134,253517969~879309,256862597~5111784,264933973~599451,270407338~3576644,297168147~5073715,302702098~43268,310505352~7387329,326427340~4602689,340517251~4565791,350426872~8101403,381207933~6058720,387676088~5905595,399030489~1461454] 2024-09-13T00:51:21.515 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to 16 diff [838088~7898100,25074977~846928,54292152~4635885,77484618~8720438,86388736~4645607,94079298~566015,126832157~11370482,143476416~7974968,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185779876~9040510,201287152~2371737,221082790~7317719,247799016~2251538,253517969~879309,256862597~5111784,264933973~599451,270407338~3576644,297168147~5073715,302702098~43268,310505352~7387329,326427340~4602689,340517251~4565791,348003072~10525203,381207933~6058720,387676088~5905595,399030489~1461454] 2024-09-13T00:51:21.515 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,25074977~846928,54292152~4635885,77484618~8720438,86388736~4645607,94079298~566015,126832157~11370482,143476416~7974968,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185779876~9040510,201287152~2371737,221082790~7317719,247799016~2251538,253517969~879309,256862597~5111784,264933973~599451,270407338~3576644,297168147~5073715,302702098~43268,310505352~7387329,326427340~4602689,340517251~4565791,348003072~10525203,381207933~6058720,387676088~5905595,399030489~1461454] 2024-09-13T00:51:21.522 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,25074977~846928,54292152~4635885,77484618~8720438,86388736~4645607,94079298~566015,126832157~11370482,142606336~8845048,155189248~7198765,170779539~568367,171824257~4101766,176889496~3521015,185779876~9040510,201224587~2434302,221082790~7317719,247799016~2251538,253517969~879309,256862597~5111784,264933973~599451,270407338~3576644,297168147~5073715,302702098~43268,310505352~7387329,326427340~4602689,340517251~4565791,348003072~10525203,381207933~6058720,387676088~5905595,399030489~1461454] 2024-09-13T00:51:21.522 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to 17 diff [838088~7898100,25074977~846928,54292152~4635885,77484618~8720438,86388736~4645607,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~7317719,244434952~6847214,253517969~879309,256862597~5111784,264793518~2809034,270407338~3576644,276169467~2517149,297168147~5073715,302702098~43268,310505352~7387329,326427340~4602689,340517251~4565791,348003072~10525203,379036633~8230020,387676088~5905595,399030489~1461454] 2024-09-13T00:51:21.522 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,25074977~846928,54292152~4635885,77484618~8720438,86388736~4645607,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~7317719,244434952~6847214,253517969~879309,256862597~5111784,264793518~2809034,270407338~3576644,276169467~2517149,297168147~5073715,302702098~43268,310505352~7387329,326427340~4602689,340517251~4565791,348003072~10525203,379036633~8230020,387676088~5905595,399030489~1461454] 2024-09-13T00:51:21.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:21 smithi077 bash[80681]: cluster 2024-09-13T00:51:20.248613+0000 mon.a (mon.0) 1410 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2024-09-13T00:51:21.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:21 smithi077 bash[83127]: cluster 2024-09-13T00:51:20.248613+0000 mon.a (mon.0) 1410 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2024-09-13T00:51:21.529 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,25074977~846928,54292152~4635885,77484618~8720438,86388736~4645607,94079298~566015,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~7317719,244434952~6847214,253517969~879309,256862597~5111784,264793518~2809034,270407338~3576644,276169467~2517149,297168147~5073715,302702098~43268,310505352~7387329,326427340~4602689,340517251~4565791,348003072~10525203,379036633~8230020,387676088~5905595,399030489~1461454] 2024-09-13T00:51:21.529 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to 18 diff [838088~7898100,25074977~846928,27408852~677832,54292152~4635885,65799589~3651744,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~11679059,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~9495859,297168147~5073715,302702098~43268,310505352~7387329,326427340~4602689,340517251~4565791,348003072~10525203,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:21.529 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,25074977~846928,27408852~677832,54292152~4635885,65799589~3651744,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~11679059,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~9495859,297168147~5073715,302702098~43268,310505352~7387329,326427340~4602689,340517251~4565791,348003072~10525203,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:21.536 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,25074977~846928,27408852~677832,54292152~4635885,65799589~3651744,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~11679059,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~9495859,297168147~5073715,302702098~43268,310505352~7387329,326427340~4602689,340517251~4565791,348003072~10525203,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:21.536 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to 19 diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:21.536 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:21.543 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:21.543 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to 14 diff [81835783~4369273,86388736~4645607,94079298~566015,126832157~6964267,171824257~4101766,189320558~5499828,221082790~7087134,253517969~879309,310505352~7387329,350426872~5051294] 2024-09-13T00:51:21.543 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [81835783~4369273,86388736~4645607,94079298~566015,126832157~6964267,171824257~4101766,189320558~5499828,221082790~7087134,253517969~879309,310505352~7387329,350426872~5051294] 2024-09-13T00:51:21.550 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [81835783~4369273,86388736~4645607,94079298~566015,126832157~6964267,171824257~4101766,189320558~5499828,221082790~7087134,253517969~879309,310505352~7387329,350426872~5051294] 2024-09-13T00:51:21.550 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to 15 diff [1657775~7078413,77484618~8720438,86388736~4645607,94079298~566015,126832157~6964267,143476416~7974968,171824257~4101766,189320558~5499828,201287152~2371737,221082790~7087134,253517969~879309,256862597~5111784,270407338~3576644,302702098~43268,310505352~7387329,326427340~4602689,350426872~5051294,399030489~1461454] 2024-09-13T00:51:21.550 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1657775~7078413,77484618~8720438,86388736~4645607,94079298~566015,126832157~6964267,143476416~7974968,171824257~4101766,189320558~5499828,201287152~2371737,221082790~7087134,253517969~879309,256862597~5111784,270407338~3576644,302702098~43268,310505352~7387329,326427340~4602689,350426872~5051294,399030489~1461454] 2024-09-13T00:51:21.556 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [1657775~7078413,77484618~8720438,86388736~4645607,94079298~566015,126832157~6964267,142606336~8845048,171824257~4101766,189320558~5499828,201224587~2434302,221082790~7087134,253517969~879309,256862597~5111784,270407338~3576644,302702098~43268,310505352~7387329,326427340~4602689,350426872~5051294,399030489~1461454] 2024-09-13T00:51:21.556 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to 16 diff [838088~7898100,25074977~846928,77484618~8720438,86388736~4645607,94079298~566015,126832157~11370482,143476416~7974968,158413680~3974333,171824257~4101766,185779876~9040510,201287152~2371737,221082790~7317719,247799016~2251538,253517969~879309,256862597~5111784,270407338~3576644,302702098~43268,310505352~7387329,326427340~4602689,342946181~1901333,348003072~7475094,399030489~1461454] 2024-09-13T00:51:21.556 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,25074977~846928,77484618~8720438,86388736~4645607,94079298~566015,126832157~11370482,143476416~7974968,158413680~3974333,171824257~4101766,185779876~9040510,201287152~2371737,221082790~7317719,247799016~2251538,253517969~879309,256862597~5111784,270407338~3576644,302702098~43268,310505352~7387329,326427340~4602689,342946181~1901333,348003072~7475094,399030489~1461454] 2024-09-13T00:51:21.563 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,25074977~846928,77484618~8720438,86388736~4645607,94079298~566015,126832157~11370482,142606336~8845048,155189248~7198765,171824257~4101766,185779876~9040510,201224587~2434302,221082790~7317719,247799016~2251538,253517969~879309,256862597~5111784,270407338~3576644,302702098~43268,310505352~7387329,326427340~4602689,342946181~1901333,348003072~7475094,399030489~1461454] 2024-09-13T00:51:21.563 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to 17 diff [838088~7898100,25074977~846928,77484618~8720438,86388736~4645607,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,171824257~4101766,185471528~9348858,201287152~16210199,221082790~7317719,244434952~6847214,253517969~879309,256862597~5111784,264793518~2809034,270407338~3576644,276169467~2517149,302702098~43268,310505352~7387329,326427340~4602689,342946181~1901333,348003072~7475094,379036633~2377445,399030489~1461454] 2024-09-13T00:51:21.563 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,25074977~846928,77484618~8720438,86388736~4645607,94079298~566015,126832157~11370482,143476416~14865258,158413680~3974333,171824257~4101766,185471528~9348858,201287152~16210199,221082790~7317719,244434952~6847214,253517969~879309,256862597~5111784,264793518~2809034,270407338~3576644,276169467~2517149,302702098~43268,310505352~7387329,326427340~4602689,342946181~1901333,348003072~7475094,379036633~2377445,399030489~1461454] 2024-09-13T00:51:21.569 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,25074977~846928,77484618~8720438,86388736~4645607,94079298~566015,126832157~11370482,142606336~19781677,171824257~4101766,185471528~9348858,201224587~16272764,221082790~7317719,244434952~6847214,253517969~879309,256862597~5111784,264793518~2809034,270407338~3576644,276169467~2517149,302702098~43268,310505352~7387329,326427340~4602689,342946181~1901333,348003072~7475094,379036633~2377445,399030489~1461454] 2024-09-13T00:51:21.569 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to 18 diff [838088~7898100,25074977~846928,27408852~677832,65799589~3651744,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,171824257~4101766,185471528~9348858,201287152~16210199,221082790~11679059,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~9495859,302702098~43268,310505352~7387329,326427340~4602689,342946181~1901333,348003072~7475094,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:21.570 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,25074977~846928,27408852~677832,65799589~3651744,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,171824257~4101766,185471528~9348858,201287152~16210199,221082790~11679059,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~9495859,302702098~43268,310505352~7387329,326427340~4602689,342946181~1901333,348003072~7475094,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:21.576 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,25074977~846928,27408852~677832,65799589~3651744,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,171824257~4101766,185471528~9348858,201224587~16272764,221082790~11679059,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~9495859,302702098~43268,310505352~7387329,326427340~4602689,342946181~1901333,348003072~7475094,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:21.576 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to 19 diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,171824257~4101766,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:21.576 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,171824257~4101766,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:21.582 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,171824257~4101766,185471528~9348858,201224587~16272764,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:21.583 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to 15 diff [1657775~7078413,77484618~5025855,143476416~7974968,201287152~2371737,256862597~5111784,270407338~3576644,302702098~43268,326427340~4602689,399030489~1461454] 2024-09-13T00:51:21.583 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [1657775~7078413,77484618~5025855,143476416~7974968,201287152~2371737,256862597~5111784,270407338~3576644,302702098~43268,326427340~4602689,399030489~1461454] 2024-09-13T00:51:21.589 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [1657775~7078413,77484618~5025855,142606336~8845048,201224587~2434302,256862597~5111784,270407338~3576644,302702098~43268,326427340~4602689,399030489~1461454] 2024-09-13T00:51:21.589 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to 16 diff [838088~7898100,25074977~846928,77484618~5025855,133094772~5107867,143476416~7974968,158413680~3974333,185779876~5370913,201287152~2371737,227470207~930302,247799016~2251538,256862597~5111784,270407338~3576644,302702098~43268,326427340~4602689,342946181~1901333,348003072~3151850,399030489~1461454] 2024-09-13T00:51:21.589 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,25074977~846928,77484618~5025855,133094772~5107867,143476416~7974968,158413680~3974333,185779876~5370913,201287152~2371737,227470207~930302,247799016~2251538,256862597~5111784,270407338~3576644,302702098~43268,326427340~4602689,342946181~1901333,348003072~3151850,399030489~1461454] 2024-09-13T00:51:21.596 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,25074977~846928,77484618~5025855,133094772~5107867,142606336~8845048,155189248~7198765,185779876~5370913,201224587~2434302,227470207~930302,247799016~2251538,256862597~5111784,270407338~3576644,302702098~43268,326427340~4602689,342946181~1901333,348003072~3151850,399030489~1461454] 2024-09-13T00:51:21.597 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to 17 diff [838088~7898100,25074977~846928,77484618~5025855,133094772~5107867,143476416~14865258,158413680~3974333,185471528~5679261,201287152~16210199,227470207~930302,244434952~6847214,256862597~5111784,264793518~2809034,270407338~3576644,276169467~2517149,302702098~43268,326427340~4602689,342946181~1901333,348003072~7454562,379036633~2377445,399030489~1461454] 2024-09-13T00:51:21.597 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,25074977~846928,77484618~5025855,133094772~5107867,143476416~14865258,158413680~3974333,185471528~5679261,201287152~16210199,227470207~930302,244434952~6847214,256862597~5111784,264793518~2809034,270407338~3576644,276169467~2517149,302702098~43268,326427340~4602689,342946181~1901333,348003072~7454562,379036633~2377445,399030489~1461454] 2024-09-13T00:51:21.604 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:21 smithi088 bash[68960]: cluster 2024-09-13T00:51:20.248613+0000 mon.a (mon.0) 1410 : cluster [DBG] osdmap e878: 8 total, 8 up, 8 in 2024-09-13T00:51:21.604 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,25074977~846928,77484618~5025855,133094772~5107867,142606336~19781677,185471528~5679261,201224587~16272764,227470207~930302,244434952~6847214,256862597~5111784,264793518~2809034,270407338~3576644,276169467~2517149,302702098~43268,326427340~4602689,342946181~1901333,348003072~7454562,379036633~2377445,399030489~1461454] 2024-09-13T00:51:21.605 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to 18 diff [838088~7898100,25074977~846928,27408852~677832,65799589~3651744,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,143476416~14865258,158413680~3974333,185471528~5679261,201287152~16210199,221102177~11659672,244434952~6847214,256862597~5111784,264793518~9190464,276169467~9495859,302702098~43268,326427340~4602689,342946181~1901333,348003072~7454562,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:21.605 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,25074977~846928,27408852~677832,65799589~3651744,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,143476416~14865258,158413680~3974333,185471528~5679261,201287152~16210199,221102177~11659672,244434952~6847214,256862597~5111784,264793518~9190464,276169467~9495859,302702098~43268,326427340~4602689,342946181~1901333,348003072~7454562,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:21.609 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,25074977~846928,27408852~677832,65799589~3651744,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,142606336~19781677,185471528~5679261,201224587~16272764,221102177~11659672,244434952~6847214,256862597~5111784,264793518~9190464,276169467~9495859,302702098~43268,326427340~4602689,342946181~1901333,348003072~7454562,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:21.610 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to 19 diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,143476416~14865258,158413680~3974333,185471528~5679261,201287152~16210199,221102177~12483070,244434952~6847214,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:21.610 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,143476416~14865258,158413680~3974333,185471528~5679261,201287152~16210199,221102177~12483070,244434952~6847214,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:21.617 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,142606336~19781677,185471528~5679261,201224587~16272764,221102177~12483070,244434952~6847214,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:21.617 INFO:tasks.workunit.client.0.smithi077.stdout:from 15 to 16 diff [838088~2598743,25074977~846928,133094772~5107867,158413680~3974333,185779876~5370913,227470207~930302,247799016~2251538,342946181~1901333,348003072~3151850] 2024-09-13T00:51:21.617 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~2598743,25074977~846928,133094772~5107867,158413680~3974333,185779876~5370913,227470207~930302,247799016~2251538,342946181~1901333,348003072~3151850] 2024-09-13T00:51:21.622 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~2598743,25074977~846928,133094772~5107867,155189248~7198765,185779876~5370913,227470207~930302,247799016~2251538,342946181~1901333,348003072~3151850] 2024-09-13T00:51:21.623 INFO:tasks.workunit.client.0.smithi077.stdout:from 15 to 17 diff [838088~2598743,25074977~846928,133094772~5107867,151225375~7116299,158413680~3974333,185471528~5679261,203062023~14435328,227470207~930302,244434952~6847214,264793518~2809034,276169467~2517149,342946181~1901333,348003072~7454562,379036633~2377445] 2024-09-13T00:51:21.623 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~2598743,25074977~846928,133094772~5107867,151225375~7116299,158413680~3974333,185471528~5679261,203062023~14435328,227470207~930302,244434952~6847214,264793518~2809034,276169467~2517149,342946181~1901333,348003072~7454562,379036633~2377445] 2024-09-13T00:51:21.629 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~2598743,25074977~846928,133094772~5107867,151225375~11162638,185471528~5679261,203062023~14435328,227470207~930302,244434952~6847214,264793518~2809034,276169467~2517149,342946181~1901333,348003072~7454562,379036633~2377445] 2024-09-13T00:51:21.629 INFO:tasks.workunit.client.0.smithi077.stdout:from 15 to 18 diff [838088~2598743,25074977~846928,27408852~677832,65799589~3651744,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,151225375~7116299,158413680~3974333,185471528~5679261,203062023~14435328,221102177~11659672,244434952~6847214,264793518~6506145,276169467~9495859,342946181~1901333,348003072~7454562,379036633~2377445,387264267~95069] 2024-09-13T00:51:21.629 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~2598743,25074977~846928,27408852~677832,65799589~3651744,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,151225375~7116299,158413680~3974333,185471528~5679261,203062023~14435328,221102177~11659672,244434952~6847214,264793518~6506145,276169467~9495859,342946181~1901333,348003072~7454562,379036633~2377445,387264267~95069] 2024-09-13T00:51:21.635 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~2598743,25074977~846928,27408852~677832,65799589~3651744,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,151225375~11162638,185471528~5679261,203062023~14435328,221102177~11659672,244434952~6847214,264793518~6506145,276169467~9495859,342946181~1901333,348003072~7454562,379036633~2377445,387264267~95069] 2024-09-13T00:51:21.635 INFO:tasks.workunit.client.0.smithi077.stdout:from 15 to 19 diff [838088~2598743,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,147709657~1438140,151225375~7116299,158413680~3974333,185471528~5679261,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:21.635 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~2598743,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,147709657~1438140,151225375~7116299,158413680~3974333,185471528~5679261,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:21.642 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~2598743,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,147709657~1438140,151225375~11162638,185471528~5679261,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:21.642 INFO:tasks.workunit.client.0.smithi077.stdout:from 16 to 17 diff [151225375~7116299,185471528~3974527,203062023~14435328,244434952~6847214,264793518~2809034,276169467~2517149,349238425~6219209,379036633~2377445] 2024-09-13T00:51:21.642 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [151225375~7116299,185471528~3974527,203062023~14435328,244434952~6847214,264793518~2809034,276169467~2517149,349238425~6219209,379036633~2377445] 2024-09-13T00:51:21.647 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [151225375~7116299,185471528~3974527,203062023~14435328,244434952~6847214,264793518~2809034,276169467~2517149,349238425~6219209,379036633~2377445] 2024-09-13T00:51:21.647 INFO:tasks.workunit.client.0.smithi077.stdout:from 16 to 18 diff [27408852~677832,65799589~3651744,83698324~943169,104340251~6346965,112314922~3751122,151225375~7116299,185471528~3974527,203062023~14435328,221102177~11659672,244434952~6847214,264793518~6506145,276169467~9495859,349238425~6219209,379036633~2377445,387264267~95069] 2024-09-13T00:51:21.648 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [27408852~677832,65799589~3651744,83698324~943169,104340251~6346965,112314922~3751122,151225375~7116299,185471528~3974527,203062023~14435328,221102177~11659672,244434952~6847214,264793518~6506145,276169467~9495859,349238425~6219209,379036633~2377445,387264267~95069] 2024-09-13T00:51:21.653 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [27408852~677832,65799589~3651744,83698324~943169,104340251~6346965,112314922~3751122,151225375~7116299,185471528~3974527,203062023~14435328,221102177~11659672,244434952~6847214,264793518~6506145,276169467~9495859,349238425~6219209,379036633~2377445,387264267~95069] 2024-09-13T00:51:21.654 INFO:tasks.workunit.client.0.smithi077.stdout:from 16 to 19 diff [15367686~2169275,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,147709657~1438140,151225375~7116299,185471528~3974527,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:21.654 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [15367686~2169275,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,147709657~1438140,151225375~7116299,185471528~3974527,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:21.661 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [13551242~3985719,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,147709657~1438140,151225375~7116299,185471528~3974527,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:21.661 INFO:tasks.workunit.client.0.smithi077.stdout:from 17 to 18 diff [27408852~677832,65799589~3651744,83698324~943169,104340251~6346965,112314922~3751122,221102177~11659672,265911692~5387971,278501790~7163536,387264267~95069] 2024-09-13T00:51:21.661 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [27408852~677832,65799589~3651744,83698324~943169,104340251~6346965,112314922~3751122,221102177~11659672,265911692~5387971,278501790~7163536,387264267~95069] 2024-09-13T00:51:21.668 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [27408852~677832,65799589~3651744,83698324~943169,104340251~6346965,112314922~3751122,221102177~11659672,265911692~5387971,278501790~7163536,387264267~95069] 2024-09-13T00:51:21.668 INFO:tasks.workunit.client.0.smithi077.stdout:from 17 to 19 diff [15367686~2169275,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,147709657~1438140,221102177~12483070,265911692~5387971,278501790~8156903,322349784~3676554,347057573~11931083,387264267~95069] 2024-09-13T00:51:21.668 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [15367686~2169275,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,147709657~1438140,221102177~12483070,265911692~5387971,278501790~8156903,322349784~3676554,347057573~11931083,387264267~95069] 2024-09-13T00:51:21.674 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [13551242~3985719,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,147709657~1438140,221102177~12483070,265911692~5387971,278501790~8156903,322349784~3676554,347057573~11931083,387264267~95069] 2024-09-13T00:51:21.674 INFO:tasks.workunit.client.0.smithi077.stdout:from 18 to 19 diff [15367686~2169275,42505574~5876247,69896487~3475498,147709657~1438140,222453969~178143,227622941~5962306,283145518~3513175,322349784~3676554,347057573~11931083] 2024-09-13T00:51:21.675 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [15367686~2169275,42505574~5876247,69896487~3475498,147709657~1438140,222453969~178143,227622941~5962306,283145518~3513175,322349784~3676554,347057573~11931083] 2024-09-13T00:51:21.681 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [13551242~3985719,42505574~5876247,69896487~3475498,147709657~1438140,222453969~178143,227622941~5962306,283145518~3513175,322349784~3676554,347057573~11931083] 2024-09-13T00:51:22.267 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.267 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.316 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:22.316 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.316 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.331 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:22.331 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.331 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.345 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:22.345 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.345 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.358 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~10489226,288350280~14950455,306184192~38898850,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:22.358 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.359 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.370 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:22.370 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.370 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.380 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.380 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.380 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.390 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~15341662,237211987~14070179,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~11708489,320232816~12541524,335206066~1129952,336808173~8274869,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.390 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.390 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.400 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,251658240~10766803,264304297~10571331,276169467~10489226,295560575~7612635,306184192~11708489,322349784~10424556,335206066~1129952,336808173~8274869,345197613~13791043,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.400 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.400 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.408 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,37748736~10633085,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,306184192~11708489,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.408 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.408 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.416 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~9411042,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.416 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to HEAD diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.416 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.423 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,185471528~9348858,195924737~21572614,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.423 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.423 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.431 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:22.431 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:22.431 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:22.438 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:22.438 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,171824257~4101766,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:22.439 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,171824257~4101766,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:22.446 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,171824257~4101766,185471528~9348858,201224587~16272764,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:22.446 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,143476416~14865258,158413680~3974333,185471528~5679261,201287152~16210199,221102177~12483070,244434952~6847214,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:22.446 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,143476416~14865258,158413680~3974333,185471528~5679261,201287152~16210199,221102177~12483070,244434952~6847214,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:22.453 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,142606336~19781677,185471528~5679261,201224587~16272764,221102177~12483070,244434952~6847214,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:22.453 INFO:tasks.workunit.client.0.smithi077.stdout:from 15 to HEAD diff [838088~2598743,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,147709657~1438140,151225375~7116299,158413680~3974333,185471528~5679261,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:22.453 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~2598743,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,147709657~1438140,151225375~7116299,158413680~3974333,185471528~5679261,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:22.460 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~2598743,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,147709657~1438140,151225375~11162638,185471528~5679261,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:22.460 INFO:tasks.workunit.client.0.smithi077.stdout:from 16 to HEAD diff [15367686~2169275,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,147709657~1438140,151225375~7116299,185471528~3974527,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:22.460 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [15367686~2169275,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,147709657~1438140,151225375~7116299,185471528~3974527,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:22.466 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [13551242~3985719,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,147709657~1438140,151225375~7116299,185471528~3974527,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:22.466 INFO:tasks.workunit.client.0.smithi077.stdout:from 17 to HEAD diff [15367686~2169275,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,147709657~1438140,221102177~12483070,265911692~5387971,278501790~8156903,322349784~3676554,347057573~11931083,387264267~95069] 2024-09-13T00:51:22.466 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [15367686~2169275,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,147709657~1438140,221102177~12483070,265911692~5387971,278501790~8156903,322349784~3676554,347057573~11931083,387264267~95069] 2024-09-13T00:51:22.472 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [13551242~3985719,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,147709657~1438140,221102177~12483070,265911692~5387971,278501790~8156903,322349784~3676554,347057573~11931083,387264267~95069] 2024-09-13T00:51:22.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:22 smithi077 bash[80681]: cluster 2024-09-13T00:51:20.831447+0000 mgr.y (mgr.25304) 464 : cluster [DBG] pgmap v962: 105 pgs: 105 active+clean; 822 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 234 KiB/s rd, 42 MiB/s wr, 362 op/s 2024-09-13T00:51:22.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:22 smithi077 bash[83127]: cluster 2024-09-13T00:51:20.831447+0000 mgr.y (mgr.25304) 464 : cluster [DBG] pgmap v962: 105 pgs: 105 active+clean; 822 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 234 KiB/s rd, 42 MiB/s wr, 362 op/s 2024-09-13T00:51:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:22 smithi088 bash[68960]: cluster 2024-09-13T00:51:20.831447+0000 mgr.y (mgr.25304) 464 : cluster [DBG] pgmap v962: 105 pgs: 105 active+clean; 822 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 234 KiB/s rd, 42 MiB/s wr, 362 op/s 2024-09-13T00:51:22.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:51:22 smithi088 bash[61791]: ts=2024-09-13T00:51:22.317Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:51:23.300 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.300 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.337 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:23.338 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.338 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.351 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:23.351 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.352 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.364 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:23.364 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.364 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.376 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~10489226,288350280~14950455,306184192~38898850,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:23.376 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.377 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.387 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:23.387 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.387 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.397 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.397 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.397 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.407 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~15341662,237211987~14070179,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~11708489,320232816~12541524,335206066~1129952,336808173~8274869,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.407 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.407 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.415 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,251658240~10766803,264304297~10571331,276169467~10489226,295560575~7612635,306184192~11708489,322349784~10424556,335206066~1129952,336808173~8274869,345197613~13791043,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.415 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.415 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.423 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,37748736~10633085,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,306184192~11708489,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.423 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.423 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.431 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~9411042,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.432 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to HEAD diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.432 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.439 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,185471528~9348858,195924737~21572614,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.439 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.440 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.447 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:23.447 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:23.447 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:23.455 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:23.455 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,171824257~4101766,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:23.455 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,171824257~4101766,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:23.462 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,171824257~4101766,185471528~9348858,201224587~16272764,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:23.462 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,143476416~14865258,158413680~3974333,185471528~5679261,201287152~16210199,221102177~12483070,244434952~6847214,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:23.463 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,143476416~14865258,158413680~3974333,185471528~5679261,201287152~16210199,221102177~12483070,244434952~6847214,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:23.469 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,142606336~19781677,185471528~5679261,201224587~16272764,221102177~12483070,244434952~6847214,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:23.469 INFO:tasks.workunit.client.0.smithi077.stdout:from 15 to HEAD diff [838088~2598743,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,147709657~1438140,151225375~7116299,158413680~3974333,185471528~5679261,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:23.469 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~2598743,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,147709657~1438140,151225375~7116299,158413680~3974333,185471528~5679261,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:23.476 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~2598743,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,147709657~1438140,151225375~11162638,185471528~5679261,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:23.476 INFO:tasks.workunit.client.0.smithi077.stdout:from 16 to HEAD diff [15367686~2169275,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,147709657~1438140,151225375~7116299,185471528~3974527,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:23.476 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [15367686~2169275,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,147709657~1438140,151225375~7116299,185471528~3974527,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:23.483 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [13551242~3985719,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,147709657~1438140,151225375~7116299,185471528~3974527,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:23.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:23 smithi077 bash[83127]: cluster 2024-09-13T00:51:22.257351+0000 mon.a (mon.0) 1411 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2024-09-13T00:51:23.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:23 smithi077 bash[80681]: cluster 2024-09-13T00:51:22.257351+0000 mon.a (mon.0) 1411 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2024-09-13T00:51:23.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:23 smithi088 bash[68960]: cluster 2024-09-13T00:51:22.257351+0000 mon.a (mon.0) 1411 : cluster [DBG] osdmap e879: 8 total, 8 up, 8 in 2024-09-13T00:51:24.311 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.312 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.343 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:24.343 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.343 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.357 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:24.357 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.357 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.369 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:24.369 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.369 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.382 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~10489226,288350280~14950455,306184192~38898850,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:24.382 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.382 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.392 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:24.392 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.393 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.403 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.403 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.403 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.412 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~15341662,237211987~14070179,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~11708489,320232816~12541524,335206066~1129952,336808173~8274869,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.412 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.412 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.421 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,251658240~10766803,264304297~10571331,276169467~10489226,295560575~7612635,306184192~11708489,322349784~10424556,335206066~1129952,336808173~8274869,345197613~13791043,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.421 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.421 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.430 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,37748736~10633085,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,306184192~11708489,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.430 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.430 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.438 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~9411042,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.438 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to HEAD diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.438 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.446 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,185471528~9348858,195924737~21572614,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.446 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.446 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.453 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:24.454 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:24.454 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:24.461 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:24.461 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,171824257~4101766,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:24.463 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,171824257~4101766,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:24.468 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,171824257~4101766,185471528~9348858,201224587~16272764,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:24.468 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,143476416~14865258,158413680~3974333,185471528~5679261,201287152~16210199,221102177~12483070,244434952~6847214,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:24.468 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,143476416~14865258,158413680~3974333,185471528~5679261,201287152~16210199,221102177~12483070,244434952~6847214,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:24.475 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,142606336~19781677,185471528~5679261,201224587~16272764,221102177~12483070,244434952~6847214,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:24.475 INFO:tasks.workunit.client.0.smithi077.stdout:from 15 to HEAD diff [838088~2598743,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,147709657~1438140,151225375~7116299,158413680~3974333,185471528~5679261,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:24.475 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~2598743,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,147709657~1438140,151225375~7116299,158413680~3974333,185471528~5679261,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:24.481 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~2598743,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,147709657~1438140,151225375~11162638,185471528~5679261,203062023~14435328,221102177~12483070,244434952~6847214,264793518~6506145,276169467~10489226,322349784~3676554,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069] 2024-09-13T00:51:24.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:24 smithi088 bash[68960]: cluster 2024-09-13T00:51:22.832220+0000 mgr.y (mgr.25304) 465 : cluster [DBG] pgmap v964: 105 pgs: 105 active+clean; 844 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 37 MiB/s wr, 372 op/s 2024-09-13T00:51:24.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:24 smithi088 bash[68960]: cluster 2024-09-13T00:51:23.281837+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2024-09-13T00:51:24.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:24 smithi077 bash[80681]: cluster 2024-09-13T00:51:22.832220+0000 mgr.y (mgr.25304) 465 : cluster [DBG] pgmap v964: 105 pgs: 105 active+clean; 844 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 37 MiB/s wr, 372 op/s 2024-09-13T00:51:24.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:24 smithi077 bash[80681]: cluster 2024-09-13T00:51:23.281837+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2024-09-13T00:51:24.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:24 smithi077 bash[83127]: cluster 2024-09-13T00:51:22.832220+0000 mgr.y (mgr.25304) 465 : cluster [DBG] pgmap v964: 105 pgs: 105 active+clean; 844 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 5.2 KiB/s rd, 37 MiB/s wr, 372 op/s 2024-09-13T00:51:24.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:24 smithi077 bash[83127]: cluster 2024-09-13T00:51:23.281837+0000 mon.a (mon.0) 1412 : cluster [DBG] osdmap e880: 8 total, 8 up, 8 in 2024-09-13T00:51:25.339 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.339 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.364 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:25.364 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.364 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.375 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:25.375 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.375 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.386 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:25.386 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.386 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.396 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~10489226,288350280~14950455,306184192~38898850,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:25.397 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.397 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.407 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:25.407 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.407 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.415 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.415 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.415 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.424 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~15341662,237211987~14070179,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~11708489,320232816~12541524,335206066~1129952,336808173~8274869,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.424 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.424 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.431 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,251658240~10766803,264304297~10571331,276169467~10489226,295560575~7612635,306184192~11708489,322349784~10424556,335206066~1129952,336808173~8274869,345197613~13791043,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.431 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.431 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.439 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,37748736~10633085,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,306184192~11708489,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.439 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.439 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.446 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~9411042,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.446 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to HEAD diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.447 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.454 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,185471528~9348858,195924737~21572614,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.454 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.454 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.461 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:25.461 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:25.461 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:25.468 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:25.468 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,171824257~4101766,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:25.468 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,171824257~4101766,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:25.474 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,171824257~4101766,185471528~9348858,201224587~16272764,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:25.474 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,143476416~14865258,158413680~3974333,185471528~5679261,201287152~16210199,221102177~12483070,244434952~6847214,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:25.474 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,143476416~14865258,158413680~3974333,185471528~5679261,201287152~16210199,221102177~12483070,244434952~6847214,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:25.481 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~5025855,83698324~943169,104340251~6346965,112314922~3751122,133094772~5107867,142606336~19781677,185471528~5679261,201224587~16272764,221102177~12483070,244434952~6847214,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:25.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:25 smithi088 bash[68960]: cluster 2024-09-13T00:51:24.280801+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2024-09-13T00:51:25.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:25 smithi077 bash[80681]: cluster 2024-09-13T00:51:24.280801+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2024-09-13T00:51:25.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:25 smithi077 bash[83127]: cluster 2024-09-13T00:51:24.280801+0000 mon.a (mon.0) 1413 : cluster [DBG] osdmap e881: 8 total, 8 up, 8 in 2024-09-13T00:51:26.345 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.346 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.377 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:26.377 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.378 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.389 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:26.389 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.390 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.401 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:26.402 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.402 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.412 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~10489226,288350280~14950455,306184192~38898850,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:26.412 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.412 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.421 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:26.421 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.422 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.430 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.430 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.430 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.438 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~15341662,237211987~14070179,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~11708489,320232816~12541524,335206066~1129952,336808173~8274869,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.438 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.438 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.446 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,251658240~10766803,264304297~10571331,276169467~10489226,295560575~7612635,306184192~11708489,322349784~10424556,335206066~1129952,336808173~8274869,345197613~13791043,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.446 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.446 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.454 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,37748736~10633085,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,306184192~11708489,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.454 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.454 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.461 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~9411042,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.461 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to HEAD diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.462 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.469 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,185471528~9348858,195924737~21572614,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.469 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.469 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.476 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:26.476 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:26.477 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:26.484 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:26.484 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,171824257~4101766,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:26.484 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,171824257~4101766,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:26.490 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,171824257~4101766,185471528~9348858,201224587~16272764,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,342946181~1901333,347057573~11931083,379036633~2377445,387264267~95069,399030489~1461454] 2024-09-13T00:51:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:26 smithi088 bash[68960]: cluster 2024-09-13T00:51:24.833267+0000 mgr.y (mgr.25304) 466 : cluster [DBG] pgmap v967: 105 pgs: 105 active+clean; 857 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 190 KiB/s rd, 9.8 MiB/s wr, 4.93k op/s 2024-09-13T00:51:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:26 smithi088 bash[68960]: cluster 2024-09-13T00:51:25.304884+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2024-09-13T00:51:26.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:26 smithi077 bash[80681]: cluster 2024-09-13T00:51:24.833267+0000 mgr.y (mgr.25304) 466 : cluster [DBG] pgmap v967: 105 pgs: 105 active+clean; 857 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 190 KiB/s rd, 9.8 MiB/s wr, 4.93k op/s 2024-09-13T00:51:26.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:26 smithi077 bash[80681]: cluster 2024-09-13T00:51:25.304884+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2024-09-13T00:51:26.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:26 smithi077 bash[83127]: cluster 2024-09-13T00:51:24.833267+0000 mgr.y (mgr.25304) 466 : cluster [DBG] pgmap v967: 105 pgs: 105 active+clean; 857 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 190 KiB/s rd, 9.8 MiB/s wr, 4.93k op/s 2024-09-13T00:51:26.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:26 smithi077 bash[83127]: cluster 2024-09-13T00:51:25.304884+0000 mon.a (mon.0) 1414 : cluster [DBG] osdmap e882: 8 total, 8 up, 8 in 2024-09-13T00:51:27.383 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.383 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.405 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:27.406 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.406 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.416 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:27.417 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.417 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.426 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:27.426 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.426 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.436 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~10489226,288350280~14950455,306184192~38898850,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:27.436 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.436 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.444 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:27.444 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.445 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.452 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.452 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.453 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.459 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~15341662,237211987~14070179,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~11708489,320232816~12541524,335206066~1129952,336808173~8274869,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.460 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.460 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.467 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,251658240~10766803,264304297~10571331,276169467~10489226,295560575~7612635,306184192~11708489,322349784~10424556,335206066~1129952,336808173~8274869,345197613~13791043,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.467 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.467 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.474 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,37748736~10633085,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,306184192~11708489,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.475 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.475 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.481 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~9411042,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.481 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to HEAD diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.482 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.488 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,185471528~9348858,195924737~21572614,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.488 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.488 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.495 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:27.495 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:27.495 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:27.501 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~8720438,86388736~4645607,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~12502457,244434952~6847214,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~4602689,340517251~4565791,347057573~11931083,379036633~8322703,387676088~5905595,399030489~1461454] 2024-09-13T00:51:27.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:27 smithi088 bash[68960]: cluster 2024-09-13T00:51:26.313402+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2024-09-13T00:51:27.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:27 smithi077 bash[80681]: cluster 2024-09-13T00:51:26.313402+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2024-09-13T00:51:27.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:27 smithi077 bash[83127]: cluster 2024-09-13T00:51:26.313402+0000 mon.a (mon.0) 1415 : cluster [DBG] osdmap e883: 8 total, 8 up, 8 in 2024-09-13T00:51:28.385 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.385 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.410 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:28.410 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.410 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.420 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:28.421 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.421 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.433 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:28.433 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.433 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.443 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~10489226,288350280~14950455,306184192~38898850,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:28.443 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.443 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.452 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:28.453 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.453 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.462 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.462 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.462 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.470 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~15341662,237211987~14070179,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~11708489,320232816~12541524,335206066~1129952,336808173~8274869,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.470 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.470 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.478 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,251658240~10766803,264304297~10571331,276169467~10489226,295560575~7612635,306184192~11708489,322349784~10424556,335206066~1129952,336808173~8274869,345197613~13791043,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.478 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.478 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.484 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,37748736~10633085,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,306184192~11708489,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.484 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.485 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.492 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~9411042,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.492 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to HEAD diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.492 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.500 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,185471528~9348858,195924737~21572614,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.500 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to HEAD diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.500 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.506 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,25074977~846928,27408852~677832,32229647~1744274,42505574~5876247,54292152~4635885,65799589~3651744,69896487~3475498,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,170779539~568367,171824257~4101766,176889496~3521015,185471528~9348858,201224587~16272764,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~9190464,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:28.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:28 smithi088 bash[68960]: cluster 2024-09-13T00:51:26.834089+0000 mgr.y (mgr.25304) 467 : cluster [DBG] pgmap v970: 105 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 102 active+clean; 852 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 214 KiB/s rd, 5.3 MiB/s wr, 5.61k op/s 2024-09-13T00:51:28.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:28 smithi088 bash[68960]: cluster 2024-09-13T00:51:27.337450+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2024-09-13T00:51:28.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:51:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:51:28] "GET /metrics HTTP/1.1" 200 36701 "" "Prometheus/2.43.0" 2024-09-13T00:51:28.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:28 smithi077 bash[80681]: cluster 2024-09-13T00:51:26.834089+0000 mgr.y (mgr.25304) 467 : cluster [DBG] pgmap v970: 105 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 102 active+clean; 852 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 214 KiB/s rd, 5.3 MiB/s wr, 5.61k op/s 2024-09-13T00:51:28.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:28 smithi077 bash[80681]: cluster 2024-09-13T00:51:27.337450+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2024-09-13T00:51:28.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:28 smithi077 bash[83127]: cluster 2024-09-13T00:51:26.834089+0000 mgr.y (mgr.25304) 467 : cluster [DBG] pgmap v970: 105 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 102 active+clean; 852 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 214 KiB/s rd, 5.3 MiB/s wr, 5.61k op/s 2024-09-13T00:51:28.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:28 smithi077 bash[83127]: cluster 2024-09-13T00:51:27.337450+0000 mon.a (mon.0) 1416 : cluster [DBG] osdmap e884: 8 total, 8 up, 8 in 2024-09-13T00:51:29.405 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.406 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.429 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:29.429 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.429 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.439 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:29.439 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.439 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.448 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:29.448 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.449 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.457 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~10489226,288350280~14950455,306184192~38898850,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:29.457 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.457 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.465 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:29.465 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.465 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.473 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.473 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.473 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.480 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~15341662,237211987~14070179,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~11708489,320232816~12541524,335206066~1129952,336808173~8274869,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.480 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.480 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.487 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,251658240~10766803,264304297~10571331,276169467~10489226,295560575~7612635,306184192~11708489,322349784~10424556,335206066~1129952,336808173~8274869,345197613~13791043,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.487 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.487 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.494 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,37748736~10633085,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,306184192~11708489,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.494 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.494 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.501 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~9411042,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.501 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to HEAD diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.501 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~7898100,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185471528~9348858,195924737~5299850,201287152~16210199,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.508 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [838088~7898100,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~5993847,42505574~5876247,54292152~4635885,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,185471528~9348858,195924737~21572614,221082790~12502457,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~3676554,326427340~6347000,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:29.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:29 smithi077 bash[80681]: cluster 2024-09-13T00:51:28.336988+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2024-09-13T00:51:29.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:29 smithi077 bash[80681]: audit 2024-09-13T00:51:28.426598+0000 mon.a (mon.0) 1418 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:29.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:29 smithi077 bash[83127]: cluster 2024-09-13T00:51:28.336988+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2024-09-13T00:51:29.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:29 smithi077 bash[83127]: audit 2024-09-13T00:51:28.426598+0000 mon.a (mon.0) 1418 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:29.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:29 smithi088 bash[68960]: cluster 2024-09-13T00:51:28.336988+0000 mon.a (mon.0) 1417 : cluster [DBG] osdmap e885: 8 total, 8 up, 8 in 2024-09-13T00:51:29.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:29 smithi088 bash[68960]: audit 2024-09-13T00:51:28.426598+0000 mon.a (mon.0) 1418 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:30.392 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.392 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.427 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:30.427 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.427 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.436 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:30.436 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.436 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.443 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:30.443 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.444 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.451 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~10489226,288350280~14950455,306184192~38898850,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:30.451 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.451 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.458 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:30.459 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.459 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.466 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.466 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.466 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.473 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~15341662,237211987~14070179,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~11708489,320232816~12541524,335206066~1129952,336808173~8274869,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.473 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.473 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.480 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,251658240~10766803,264304297~10571331,276169467~10489226,295560575~7612635,306184192~11708489,322349784~10424556,335206066~1129952,336808173~8274869,345197613~13791043,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.480 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.480 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.486 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,37748736~10633085,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,306184192~11708489,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.486 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.487 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~8572954,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.493 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~9411042,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,42505574~5876247,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,241666235~9615931,253517969~879309,256862597~5111784,264793518~10082110,276169467~10489226,297168147~5073715,302702098~43268,310505352~7387329,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,379036633~14545050,399030489~1461454,406071955~8147490] 2024-09-13T00:51:30.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:30 smithi077 bash[80681]: cluster 2024-09-13T00:51:28.834911+0000 mgr.y (mgr.25304) 468 : cluster [DBG] pgmap v973: 105 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 102 active+clean; 834 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 135 KiB/s rd, 6.2 KiB/s wr, 1.03k op/s 2024-09-13T00:51:30.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:30 smithi077 bash[80681]: cluster 2024-09-13T00:51:29.359721+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2024-09-13T00:51:30.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:30 smithi077 bash[83127]: cluster 2024-09-13T00:51:28.834911+0000 mgr.y (mgr.25304) 468 : cluster [DBG] pgmap v973: 105 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 102 active+clean; 834 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 135 KiB/s rd, 6.2 KiB/s wr, 1.03k op/s 2024-09-13T00:51:30.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:30 smithi077 bash[83127]: cluster 2024-09-13T00:51:29.359721+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2024-09-13T00:51:30.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:30 smithi088 bash[68960]: cluster 2024-09-13T00:51:28.834911+0000 mgr.y (mgr.25304) 468 : cluster [DBG] pgmap v973: 105 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 102 active+clean; 834 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 135 KiB/s rd, 6.2 KiB/s wr, 1.03k op/s 2024-09-13T00:51:30.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:30 smithi088 bash[68960]: cluster 2024-09-13T00:51:29.359721+0000 mon.a (mon.0) 1419 : cluster [DBG] osdmap e886: 8 total, 8 up, 8 in 2024-09-13T00:51:31.127 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.127 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.166 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:31.166 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.166 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.175 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:31.175 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.175 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.183 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:31.183 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.183 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.191 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~10489226,288350280~14950455,306184192~38898850,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:31.191 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.191 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.198 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:31.198 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.198 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.205 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.205 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.205 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.212 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~15341662,237211987~14070179,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~11708489,320232816~12541524,335206066~1129952,336808173~8274869,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.212 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.212 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.219 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,251658240~10766803,264304297~10571331,276169467~10489226,295560575~7612635,306184192~11708489,322349784~10424556,335206066~1129952,336808173~8274869,345197613~13791043,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.219 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.219 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,41106664~7275157,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,307269978~110347,308023339~9869342,322349784~10424556,335206066~179599,336808173~8274869,347057573~11931083,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.225 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~2994873,25074977~846928,27408852~677832,29503754~8028782,37748736~10633085,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,253517969~879309,256862597~5111784,264304297~10571331,276169467~10489226,297168147~5073715,302702098~43268,306184192~11708489,322349784~10424556,335206066~179599,336808173~8274869,345197613~13791043,377731437~19052729,399030489~1461454,406071955~8147490] 2024-09-13T00:51:31.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:31 smithi077 bash[80681]: cluster 2024-09-13T00:51:30.360974+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2024-09-13T00:51:31.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:31 smithi077 bash[80681]: cluster 2024-09-13T00:51:31.097915+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2024-09-13T00:51:31.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:31 smithi077 bash[83127]: cluster 2024-09-13T00:51:30.360974+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2024-09-13T00:51:31.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:31 smithi077 bash[83127]: cluster 2024-09-13T00:51:31.097915+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2024-09-13T00:51:31.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:31 smithi088 bash[68960]: cluster 2024-09-13T00:51:30.360974+0000 mon.a (mon.0) 1420 : cluster [DBG] osdmap e887: 8 total, 8 up, 8 in 2024-09-13T00:51:31.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:31 smithi088 bash[68960]: cluster 2024-09-13T00:51:31.097915+0000 mon.a (mon.0) 1421 : cluster [DBG] osdmap e888: 8 total, 8 up, 8 in 2024-09-13T00:51:32.124 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.125 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.171 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:32.171 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.171 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.180 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:32.180 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.180 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.188 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:32.189 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.189 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.196 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~10489226,288350280~14950455,306184192~38898850,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:32.196 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.197 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.204 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:32.204 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.204 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.211 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.211 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.211 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.219 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~15341662,237211987~14070179,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~11708489,320232816~12541524,335206066~1129952,336808173~8274869,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.219 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.219 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~6791266,176889496~3521015,185145489~9674897,195924737~5299850,201287152~16210199,218243585~15341662,237941494~13340672,253517969~879309,255387586~7037457,264304297~10571331,276169467~10489226,295560575~7612635,307269978~110347,308023339~9869342,322349784~10424556,335206066~1129952,336808173~8274869,347057573~11931083,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.225 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,54292152~4635885,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~6791266,176889496~3521015,184549376~10271010,195924737~21572614,218243585~15341662,237941494~13340672,251658240~10766803,264304297~10571331,276169467~10489226,295560575~7612635,306184192~11708489,322349784~10424556,335206066~1129952,336808173~8274869,345197613~13791043,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:32.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:32 smithi088 bash[68960]: cluster 2024-09-13T00:51:30.835570+0000 mgr.y (mgr.25304) 469 : cluster [DBG] pgmap v976: 105 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 102 active+clean; 792 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 136 KiB/s rd, 6.2 KiB/s wr, 1.63k op/s 2024-09-13T00:51:32.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:32 smithi088 bash[68960]: cluster 2024-09-13T00:51:32.100773+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2024-09-13T00:51:32.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:51:32 smithi088 bash[61791]: ts=2024-09-13T00:51:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:51:32.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:32 smithi077 bash[80681]: cluster 2024-09-13T00:51:30.835570+0000 mgr.y (mgr.25304) 469 : cluster [DBG] pgmap v976: 105 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 102 active+clean; 792 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 136 KiB/s rd, 6.2 KiB/s wr, 1.63k op/s 2024-09-13T00:51:32.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:32 smithi077 bash[80681]: cluster 2024-09-13T00:51:32.100773+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2024-09-13T00:51:32.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:32 smithi077 bash[83127]: cluster 2024-09-13T00:51:30.835570+0000 mgr.y (mgr.25304) 469 : cluster [DBG] pgmap v976: 105 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 102 active+clean; 792 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 136 KiB/s rd, 6.2 KiB/s wr, 1.63k op/s 2024-09-13T00:51:32.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:32 smithi077 bash[83127]: cluster 2024-09-13T00:51:32.100773+0000 mon.a (mon.0) 1422 : cluster [DBG] osdmap e889: 8 total, 8 up, 8 in 2024-09-13T00:51:33.134 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:33.134 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:33.179 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:33.179 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:33.179 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:33.190 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:33.190 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:33.190 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:33.200 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:33.200 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:33.200 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:33.209 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~10489226,288350280~14950455,306184192~38898850,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:33.209 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:33.209 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:33.217 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:33.217 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:33.217 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:33.224 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:33.224 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:33.224 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7275157,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~15341662,237211987~14070179,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~9869342,320232816~12541524,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:33.231 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~10633085,48987206~726746,49928767~8999270,59919134~784701,64412830~8959155,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~15341662,237211987~14070179,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~11708489,320232816~12541524,335206066~1129952,336808173~8274869,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:34.150 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:34.150 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:34.180 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:34.181 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:34.181 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:34.189 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:34.189 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:34.190 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:34.198 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:34.198 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:34.198 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:34.206 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~10489226,288350280~14950455,306184192~38898850,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:34.206 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:34.206 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:34.214 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:34.214 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:34.214 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:34.220 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~6346965,112314922~3751122,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:34 smithi077 bash[80681]: cluster 2024-09-13T00:51:32.836303+0000 mgr.y (mgr.25304) 470 : cluster [DBG] pgmap v979: 105 pgs: 105 active+clean; 787 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 1.18k op/s 2024-09-13T00:51:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:34 smithi077 bash[80681]: cluster 2024-09-13T00:51:33.107715+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2024-09-13T00:51:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:34 smithi077 bash[83127]: cluster 2024-09-13T00:51:32.836303+0000 mgr.y (mgr.25304) 470 : cluster [DBG] pgmap v979: 105 pgs: 105 active+clean; 787 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 1.18k op/s 2024-09-13T00:51:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:34 smithi077 bash[83127]: cluster 2024-09-13T00:51:33.107715+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2024-09-13T00:51:34.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:34 smithi088 bash[68960]: cluster 2024-09-13T00:51:32.836303+0000 mgr.y (mgr.25304) 470 : cluster [DBG] pgmap v979: 105 pgs: 105 active+clean; 787 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 2.5 KiB/s rd, 0 B/s wr, 1.18k op/s 2024-09-13T00:51:34.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:34 smithi088 bash[68960]: cluster 2024-09-13T00:51:33.107715+0000 mon.a (mon.0) 1423 : cluster [DBG] osdmap e890: 8 total, 8 up, 8 in 2024-09-13T00:51:35.154 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:35.154 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:35.201 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:35.202 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:35.202 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:35.211 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:35.211 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:35.211 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:35.219 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:35.220 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:35.220 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:35.227 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~10489226,288350280~14950455,306184192~38898850,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:35.227 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:35.228 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~2169275,20029799~5892106,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77484618~13549725,94079298~566015,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,295560575~7740160,307269978~110347,308023339~24751001,335206066~1129952,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:35.235 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~3985719,20029799~5892106,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~9107060,77450102~13584241,94079298~566015,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264304297~10571331,276169467~10489226,295560575~7740160,306184192~26590148,335206066~9876976,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:35.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:35 smithi077 bash[80681]: cluster 2024-09-13T00:51:34.106487+0000 mon.a (mon.0) 1424 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2024-09-13T00:51:35.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:35 smithi077 bash[83127]: cluster 2024-09-13T00:51:34.106487+0000 mon.a (mon.0) 1424 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2024-09-13T00:51:35.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:35 smithi088 bash[68960]: cluster 2024-09-13T00:51:34.106487+0000 mon.a (mon.0) 1424 : cluster [DBG] osdmap e891: 8 total, 8 up, 8 in 2024-09-13T00:51:36.129 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:36.129 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:36.161 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:36.161 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:36.161 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:36.170 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:36.170 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:36.170 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:36.177 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:36.177 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:36.177 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104340251~6346965,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,276169467~10489226,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8274869,347057573~11931083,368358100~1519512,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:36.184 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,104340251~11725793,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,276169467~10489226,288350280~14950455,306184192~38898850,345197613~13791043,368358100~1519512,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:36.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:36 smithi077 bash[80681]: cluster 2024-09-13T00:51:34.837348+0000 mgr.y (mgr.25304) 471 : cluster [DBG] pgmap v982: 105 pgs: 105 active+clean; 656 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 1.38k op/s 2024-09-13T00:51:36.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:36 smithi077 bash[80681]: cluster 2024-09-13T00:51:35.132309+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2024-09-13T00:51:36.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:36 smithi077 bash[80681]: cluster 2024-09-13T00:51:36.099459+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2024-09-13T00:51:36.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:36 smithi077 bash[83127]: cluster 2024-09-13T00:51:34.837348+0000 mgr.y (mgr.25304) 471 : cluster [DBG] pgmap v982: 105 pgs: 105 active+clean; 656 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 1.38k op/s 2024-09-13T00:51:36.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:36 smithi077 bash[83127]: cluster 2024-09-13T00:51:35.132309+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2024-09-13T00:51:36.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:36 smithi077 bash[83127]: cluster 2024-09-13T00:51:36.099459+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2024-09-13T00:51:36.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:36 smithi088 bash[68960]: cluster 2024-09-13T00:51:34.837348+0000 mgr.y (mgr.25304) 471 : cluster [DBG] pgmap v982: 105 pgs: 105 active+clean; 656 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 1.38k op/s 2024-09-13T00:51:36.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:36 smithi088 bash[68960]: cluster 2024-09-13T00:51:35.132309+0000 mon.a (mon.0) 1425 : cluster [DBG] osdmap e892: 8 total, 8 up, 8 in 2024-09-13T00:51:36.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:36 smithi088 bash[68960]: cluster 2024-09-13T00:51:36.099459+0000 mon.a (mon.0) 1426 : cluster [DBG] osdmap e893: 8 total, 8 up, 8 in 2024-09-13T00:51:37.186 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:37.187 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:37.210 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:37.210 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:37.210 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:37.219 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:37.220 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:37.220 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,15367686~10554219,27408852~677832,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,124397052~1978614,126832157~11370482,143476416~14865258,158413680~3974333,169134757~11275754,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288350280~14950455,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:37.229 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~11521143,13551242~12370663,27408852~677832,29503754~8028782,37748736~11133846,48987206~726746,49928767~8999270,59919134~784701,64412830~26621513,92274688~3369448,100663296~15402748,124397052~1978614,126832157~11370482,142606336~19781677,169134757~11275754,184549376~32947975,218243585~33038581,251658240~11501142,264241152~10634476,275274515~11384178,288350280~14950455,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:38.177 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:38.178 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:38.211 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:38.211 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:38.211 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,121899613~16303026,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,366062574~7797753,375685896~21098270,399030489~1461454,406071955~8147490] 2024-09-13T00:51:38.220 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,121899613~16303026,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,366062574~7797753,375685896~21098270,399030489~1461454,405420596~8798849] 2024-09-13T00:51:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:38 smithi077 bash[83127]: cluster 2024-09-13T00:51:36.838232+0000 mgr.y (mgr.25304) 472 : cluster [DBG] pgmap v985: 105 pgs: 105 active+clean; 646 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 1.26k op/s 2024-09-13T00:51:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:38 smithi077 bash[83127]: cluster 2024-09-13T00:51:37.142731+0000 mon.a (mon.0) 1427 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2024-09-13T00:51:38.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:38 smithi077 bash[80681]: cluster 2024-09-13T00:51:36.838232+0000 mgr.y (mgr.25304) 472 : cluster [DBG] pgmap v985: 105 pgs: 105 active+clean; 646 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 1.26k op/s 2024-09-13T00:51:38.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:38 smithi077 bash[80681]: cluster 2024-09-13T00:51:37.142731+0000 mon.a (mon.0) 1427 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2024-09-13T00:51:38.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:38 smithi088 bash[68960]: cluster 2024-09-13T00:51:36.838232+0000 mgr.y (mgr.25304) 472 : cluster [DBG] pgmap v985: 105 pgs: 105 active+clean; 646 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 5.7 KiB/s wr, 1.26k op/s 2024-09-13T00:51:38.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:38 smithi088 bash[68960]: cluster 2024-09-13T00:51:37.142731+0000 mon.a (mon.0) 1427 : cluster [DBG] osdmap e894: 8 total, 8 up, 8 in 2024-09-13T00:51:38.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:51:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:51:38] "GET /metrics HTTP/1.1" 200 36701 "" "Prometheus/2.43.0" 2024-09-13T00:51:39.196 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:39.196 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [838088~10683055,11835209~1716033,15367686~10554219,27215137~984633,29503754~8028782,41106664~7775918,48987206~726746,49928767~8999270,59919134~784701,62780271~112299,64412830~13037272,77484618~13549725,94079298~566015,94755463~888673,104204950~6482266,111226631~4839413,120022934~18179705,143476416~14865258,158413680~3974333,167226701~16924508,185145489~16079098,201287152~16210199,218243585~33038581,253517969~879309,255387586~7771796,264304297~10571331,275274515~11384178,288050963~15249772,307269978~110347,308023339~24751001,333121499~3214519,336808173~8389440,347057573~11931083,363850683~866908,365523275~8337052,375685896~22487292,399030489~1461454,406071955~8147490] 2024-09-13T00:51:39.227 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~28199770,29360128~8172408,37748736~11133846,48987206~726746,49928767~12963803,64412830~26621513,92274688~3369448,100663296~15402748,117440512~20762127,142606336~19781677,163577856~20573353,184549376~32947975,218243585~33038581,251658240~11501142,264241152~39059583,306184192~52804464,360710144~4007447,364904448~8955879,375685896~22487292,399030489~1461454,405420596~8798849] 2024-09-13T00:51:39.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:39 smithi077 bash[80681]: cluster 2024-09-13T00:51:38.145417+0000 mon.a (mon.0) 1428 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2024-09-13T00:51:39.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:39 smithi077 bash[83127]: cluster 2024-09-13T00:51:38.145417+0000 mon.a (mon.0) 1428 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2024-09-13T00:51:39.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:39 smithi088 bash[68960]: cluster 2024-09-13T00:51:38.145417+0000 mon.a (mon.0) 1428 : cluster [DBG] osdmap e895: 8 total, 8 up, 8 in 2024-09-13T00:51:40.232 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/0.DiffIterateStress (40943 ms) 2024-09-13T00:51:40.233 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/0.DiffIterateRegression6926 2024-09-13T00:51:40.233 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:51:40.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:40 smithi077 bash[80681]: cluster 2024-09-13T00:51:38.839100+0000 mgr.y (mgr.25304) 473 : cluster [DBG] pgmap v988: 105 pgs: 105 active+clean; 611 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 69 KiB/s rd, 6.0 KiB/s wr, 335 op/s 2024-09-13T00:51:40.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:40 smithi077 bash[80681]: cluster 2024-09-13T00:51:39.173650+0000 mon.a (mon.0) 1429 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2024-09-13T00:51:40.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:40 smithi077 bash[83127]: cluster 2024-09-13T00:51:38.839100+0000 mgr.y (mgr.25304) 473 : cluster [DBG] pgmap v988: 105 pgs: 105 active+clean; 611 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 69 KiB/s rd, 6.0 KiB/s wr, 335 op/s 2024-09-13T00:51:40.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:40 smithi077 bash[83127]: cluster 2024-09-13T00:51:39.173650+0000 mon.a (mon.0) 1429 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2024-09-13T00:51:40.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:40 smithi088 bash[68960]: cluster 2024-09-13T00:51:38.839100+0000 mgr.y (mgr.25304) 473 : cluster [DBG] pgmap v988: 105 pgs: 105 active+clean; 611 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 69 KiB/s rd, 6.0 KiB/s wr, 335 op/s 2024-09-13T00:51:40.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:40 smithi088 bash[68960]: cluster 2024-09-13T00:51:39.173650+0000 mon.a (mon.0) 1429 : cluster [DBG] osdmap e896: 8 total, 8 up, 8 in 2024-09-13T00:51:41.135 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 0~256 2024-09-13T00:51:41.146 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/0.DiffIterateRegression6926 (913 ms) 2024-09-13T00:51:41.146 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParent 2024-09-13T00:51:41.146 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:51:41.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:41 smithi077 bash[80681]: cluster 2024-09-13T00:51:40.182297+0000 mon.a (mon.0) 1430 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2024-09-13T00:51:41.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:41 smithi077 bash[80681]: cluster 2024-09-13T00:51:41.102293+0000 mon.a (mon.0) 1431 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2024-09-13T00:51:41.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:41 smithi077 bash[83127]: cluster 2024-09-13T00:51:40.182297+0000 mon.a (mon.0) 1430 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2024-09-13T00:51:41.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:41 smithi077 bash[83127]: cluster 2024-09-13T00:51:41.102293+0000 mon.a (mon.0) 1431 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2024-09-13T00:51:41.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:41 smithi088 bash[68960]: cluster 2024-09-13T00:51:40.182297+0000 mon.a (mon.0) 1430 : cluster [DBG] osdmap e897: 8 total, 8 up, 8 in 2024-09-13T00:51:41.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:41 smithi088 bash[68960]: cluster 2024-09-13T00:51:41.102293+0000 mon.a (mon.0) 1431 : cluster [DBG] osdmap e898: 8 total, 8 up, 8 in 2024-09-13T00:51:42.347 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 0~4194304 2024-09-13T00:51:42.347 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 4194304~4194304 2024-09-13T00:51:42.347 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 8388608~4194304 2024-09-13T00:51:42.347 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 12582912~4194304 2024-09-13T00:51:42.347 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 16777216~4194304 2024-09-13T00:51:42.418 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 0~4194304 2024-09-13T00:51:42.418 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 4194304~4194304 2024-09-13T00:51:42.418 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 8388608~2097152 2024-09-13T00:51:42.418 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 16777216~4194304 2024-09-13T00:51:42.445 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/0.DiffIterateParent (1300 ms) 2024-09-13T00:51:42.446 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/0.DiffIterateIgnoreParent 2024-09-13T00:51:42.446 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:51:42.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:42 smithi077 bash[83127]: cluster 2024-09-13T00:51:40.839760+0000 mgr.y (mgr.25304) 474 : cluster [DBG] pgmap v991: 105 pgs: 105 active+clean; 505 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 6.0 KiB/s wr, 473 op/s 2024-09-13T00:51:42.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:42 smithi077 bash[80681]: cluster 2024-09-13T00:51:40.839760+0000 mgr.y (mgr.25304) 474 : cluster [DBG] pgmap v991: 105 pgs: 105 active+clean; 505 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 6.0 KiB/s wr, 473 op/s 2024-09-13T00:51:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:42 smithi088 bash[68960]: cluster 2024-09-13T00:51:40.839760+0000 mgr.y (mgr.25304) 474 : cluster [DBG] pgmap v991: 105 pgs: 105 active+clean; 505 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 6.0 KiB/s wr, 473 op/s 2024-09-13T00:51:42.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:51:42 smithi088 bash[61791]: ts=2024-09-13T00:51:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:51:43.483 INFO:tasks.workunit.client.0.smithi077.stdout: wrote [124745~27926,4061564~71498,7508624~95571,8638361~5429,11825521~72449,13336305~97702,16238554~92166,18116292~81772,19358865~49637,19936714~12144] to clone 2024-09-13T00:51:43.485 INFO:tasks.workunit.client.0.smithi077.stdout: diff was [0~4133062,4194304~3409891,8388608~3509362,12582912~3747808,16777216~3171642] 2024-09-13T00:51:43.494 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/0.DiffIterateIgnoreParent (1048 ms) 2024-09-13T00:51:43.494 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/0.DiffIterateCallbackError 2024-09-13T00:51:43.494 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:51:43.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:43 smithi077 bash[80681]: cluster 2024-09-13T00:51:42.199731+0000 mon.a (mon.0) 1432 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2024-09-13T00:51:43.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:43 smithi077 bash[83127]: cluster 2024-09-13T00:51:42.199731+0000 mon.a (mon.0) 1432 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2024-09-13T00:51:43.545 INFO:tasks.workunit.client.0.smithi077.stdout: wrote [1058291~39446,3594507~47754,3963142~65244,4959765~89984,6769932~46982,8733922~55064,11090976~27708,14269933~7755,14713661~95548,15573595~22391] 2024-09-13T00:51:43.574 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/0.DiffIterateCallbackError (80 ms) 2024-09-13T00:51:43.574 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParentDiscard 2024-09-13T00:51:43.574 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:51:43.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:43 smithi088 bash[68960]: cluster 2024-09-13T00:51:42.199731+0000 mon.a (mon.0) 1432 : cluster [DBG] osdmap e899: 8 total, 8 up, 8 in 2024-09-13T00:51:44.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:44 smithi077 bash[80681]: cluster 2024-09-13T00:51:42.840368+0000 mgr.y (mgr.25304) 475 : cluster [DBG] pgmap v994: 105 pgs: 105 active+clean; 483 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 1023 B/s wr, 264 op/s 2024-09-13T00:51:44.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:44 smithi077 bash[80681]: cluster 2024-09-13T00:51:43.215101+0000 mon.a (mon.0) 1433 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2024-09-13T00:51:44.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:44 smithi077 bash[80681]: audit 2024-09-13T00:51:43.426977+0000 mon.a (mon.0) 1434 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:44.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:44 smithi077 bash[83127]: cluster 2024-09-13T00:51:42.840368+0000 mgr.y (mgr.25304) 475 : cluster [DBG] pgmap v994: 105 pgs: 105 active+clean; 483 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 1023 B/s wr, 264 op/s 2024-09-13T00:51:44.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:44 smithi077 bash[83127]: cluster 2024-09-13T00:51:43.215101+0000 mon.a (mon.0) 1433 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2024-09-13T00:51:44.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:44 smithi077 bash[83127]: audit 2024-09-13T00:51:43.426977+0000 mon.a (mon.0) 1434 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:44.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:44 smithi088 bash[68960]: cluster 2024-09-13T00:51:42.840368+0000 mgr.y (mgr.25304) 475 : cluster [DBG] pgmap v994: 105 pgs: 105 active+clean; 483 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 15 KiB/s rd, 1023 B/s wr, 264 op/s 2024-09-13T00:51:44.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:44 smithi088 bash[68960]: cluster 2024-09-13T00:51:43.215101+0000 mon.a (mon.0) 1433 : cluster [DBG] osdmap e900: 8 total, 8 up, 8 in 2024-09-13T00:51:44.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:44 smithi088 bash[68960]: audit 2024-09-13T00:51:43.426977+0000 mon.a (mon.0) 1434 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:45.459 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/0.DiffIterateParentDiscard (1886 ms) 2024-09-13T00:51:45.460 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnalignedSmall 2024-09-13T00:51:45.460 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:51:45.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:45 smithi077 bash[83127]: cluster 2024-09-13T00:51:44.223885+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2024-09-13T00:51:45.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:45 smithi077 bash[80681]: cluster 2024-09-13T00:51:44.223885+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2024-09-13T00:51:45.552 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 5000005~1234 2024-09-13T00:51:45.561 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnalignedSmall (101 ms) 2024-09-13T00:51:45.561 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnaligned 2024-09-13T00:51:45.561 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:51:45.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:45 smithi088 bash[68960]: cluster 2024-09-13T00:51:44.223885+0000 mon.a (mon.0) 1435 : cluster [DBG] osdmap e901: 8 total, 8 up, 8 in 2024-09-13T00:51:45.707 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 8376263~12345 2024-09-13T00:51:45.707 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 8388608~4194304 2024-09-13T00:51:45.707 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 12582912~54321 2024-09-13T00:51:45.721 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnaligned (160 ms) 2024-09-13T00:51:45.721 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStriping 2024-09-13T00:51:45.721 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:51:45.721 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/0.DiffIterateStriping (0 ms) 2024-09-13T00:51:46.109 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 11 tests from DiffIterateTest/0 (54523 ms total) 2024-09-13T00:51:46.109 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:51:46.109 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 11 tests from DiffIterateTest/1, where TypeParam = DiffIterateParams 2024-09-13T00:51:46.183 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/1.DiffIterate 2024-09-13T00:51:47.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:46 smithi077 bash[80681]: cluster 2024-09-13T00:51:44.841515+0000 mgr.y (mgr.25304) 476 : cluster [DBG] pgmap v997: 105 pgs: 105 active+clean; 505 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 9.8 MiB/s wr, 313 op/s 2024-09-13T00:51:47.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:46 smithi077 bash[80681]: cluster 2024-09-13T00:51:45.233731+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2024-09-13T00:51:47.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:46 smithi077 bash[80681]: cluster 2024-09-13T00:51:46.105859+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2024-09-13T00:51:47.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:46 smithi077 bash[83127]: cluster 2024-09-13T00:51:44.841515+0000 mgr.y (mgr.25304) 476 : cluster [DBG] pgmap v997: 105 pgs: 105 active+clean; 505 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 9.8 MiB/s wr, 313 op/s 2024-09-13T00:51:47.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:46 smithi077 bash[83127]: cluster 2024-09-13T00:51:45.233731+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2024-09-13T00:51:47.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:46 smithi077 bash[83127]: cluster 2024-09-13T00:51:46.105859+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2024-09-13T00:51:47.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:46 smithi088 bash[68960]: cluster 2024-09-13T00:51:44.841515+0000 mgr.y (mgr.25304) 476 : cluster [DBG] pgmap v997: 105 pgs: 105 active+clean; 505 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 9.8 MiB/s wr, 313 op/s 2024-09-13T00:51:47.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:46 smithi088 bash[68960]: cluster 2024-09-13T00:51:45.233731+0000 mon.a (mon.0) 1436 : cluster [DBG] osdmap e902: 8 total, 8 up, 8 in 2024-09-13T00:51:47.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:46 smithi088 bash[68960]: cluster 2024-09-13T00:51:46.105859+0000 mon.a (mon.0) 1437 : cluster [DBG] osdmap e903: 8 total, 8 up, 8 in 2024-09-13T00:51:48.128 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:51:48.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:48 smithi077 bash[80681]: cluster 2024-09-13T00:51:46.842280+0000 mgr.y (mgr.25304) 477 : cluster [DBG] pgmap v1000: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:51:48.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:48 smithi077 bash[80681]: cluster 2024-09-13T00:51:47.108727+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2024-09-13T00:51:48.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:48 smithi077 bash[80681]: audit 2024-09-13T00:51:47.110911+0000 mon.c (mon.2) 48 : audit [INF] from='client.? 172.21.15.77:0/858763259' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:51:48.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:48 smithi077 bash[80681]: audit 2024-09-13T00:51:47.112040+0000 mon.a (mon.0) 1439 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:51:48.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:48 smithi077 bash[83127]: cluster 2024-09-13T00:51:46.842280+0000 mgr.y (mgr.25304) 477 : cluster [DBG] pgmap v1000: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:51:48.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:48 smithi077 bash[83127]: cluster 2024-09-13T00:51:47.108727+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2024-09-13T00:51:48.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:48 smithi077 bash[83127]: audit 2024-09-13T00:51:47.110911+0000 mon.c (mon.2) 48 : audit [INF] from='client.? 172.21.15.77:0/858763259' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:51:48.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:48 smithi077 bash[83127]: audit 2024-09-13T00:51:47.112040+0000 mon.a (mon.0) 1439 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:51:48.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:48 smithi088 bash[68960]: cluster 2024-09-13T00:51:46.842280+0000 mgr.y (mgr.25304) 477 : cluster [DBG] pgmap v1000: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:51:48.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:48 smithi088 bash[68960]: cluster 2024-09-13T00:51:47.108727+0000 mon.a (mon.0) 1438 : cluster [DBG] osdmap e904: 8 total, 8 up, 8 in 2024-09-13T00:51:48.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:48 smithi088 bash[68960]: audit 2024-09-13T00:51:47.110911+0000 mon.c (mon.2) 48 : audit [INF] from='client.? 172.21.15.77:0/858763259' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:51:48.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:48 smithi088 bash[68960]: audit 2024-09-13T00:51:47.112040+0000 mon.a (mon.0) 1439 : audit [INF] from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-13T00:51:48.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:51:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:51:48] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.43.0" 2024-09-13T00:51:49.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:49 smithi077 bash[80681]: audit 2024-09-13T00:51:48.113273+0000 mon.a (mon.0) 1440 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:51:49.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:49 smithi077 bash[80681]: cluster 2024-09-13T00:51:48.114694+0000 mon.a (mon.0) 1441 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2024-09-13T00:51:49.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:49 smithi077 bash[83127]: audit 2024-09-13T00:51:48.113273+0000 mon.a (mon.0) 1440 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:51:49.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:49 smithi077 bash[83127]: cluster 2024-09-13T00:51:48.114694+0000 mon.a (mon.0) 1441 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2024-09-13T00:51:49.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:49 smithi088 bash[68960]: audit 2024-09-13T00:51:48.113273+0000 mon.a (mon.0) 1440 : audit [INF] from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi077-68798-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-13T00:51:49.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:49 smithi088 bash[68960]: cluster 2024-09-13T00:51:48.114694+0000 mon.a (mon.0) 1441 : cluster [DBG] osdmap e905: 8 total, 8 up, 8 in 2024-09-13T00:51:50.181 INFO:tasks.workunit.client.0.smithi077.stdout: wrote [1447467~59736,3428249~52971,4282271~82819,4773926~26942,10483853~82315,11443101~31469,14034304~45803,17455712~13593,17929690~78129,19420774~26752] 2024-09-13T00:51:50.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:50 smithi077 bash[80681]: cluster 2024-09-13T00:51:48.843045+0000 mgr.y (mgr.25304) 478 : cluster [DBG] pgmap v1003: 105 pgs: 4 creating+peering, 22 unknown, 79 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:51:50.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:50 smithi077 bash[80681]: cluster 2024-09-13T00:51:49.140265+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2024-09-13T00:51:50.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:50 smithi077 bash[83127]: cluster 2024-09-13T00:51:48.843045+0000 mgr.y (mgr.25304) 478 : cluster [DBG] pgmap v1003: 105 pgs: 4 creating+peering, 22 unknown, 79 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:51:50.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:50 smithi077 bash[83127]: cluster 2024-09-13T00:51:49.140265+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2024-09-13T00:51:50.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:50 smithi088 bash[68960]: cluster 2024-09-13T00:51:48.843045+0000 mgr.y (mgr.25304) 478 : cluster [DBG] pgmap v1003: 105 pgs: 4 creating+peering, 22 unknown, 79 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:51:50.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:50 smithi088 bash[68960]: cluster 2024-09-13T00:51:49.140265+0000 mon.a (mon.0) 1442 : cluster [DBG] osdmap e906: 8 total, 8 up, 8 in 2024-09-13T00:51:51.189 INFO:tasks.workunit.client.0.smithi077.stdout: wrote [0~4194304,8388608~12582912] 2024-09-13T00:51:51.207 INFO:tasks.workunit.client.0.smithi077.stdout: diff was [0~4194304,8388608~12582912] 2024-09-13T00:51:51.215 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/1.DiffIterate (5035 ms) 2024-09-13T00:51:51.216 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDiscard 2024-09-13T00:51:51.216 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:51:51.268 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 0~4194304 2024-09-13T00:51:51.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:51 smithi077 bash[80681]: cluster 2024-09-13T00:51:50.139256+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2024-09-13T00:51:51.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:51 smithi077 bash[83127]: cluster 2024-09-13T00:51:50.139256+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2024-09-13T00:51:51.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:51 smithi088 bash[68960]: cluster 2024-09-13T00:51:50.139256+0000 mon.a (mon.0) 1443 : cluster [DBG] osdmap e907: 8 total, 8 up, 8 in 2024-09-13T00:51:52.200 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 0~4194304 2024-09-13T00:51:52.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:52 smithi077 bash[80681]: cluster 2024-09-13T00:51:50.843763+0000 mgr.y (mgr.25304) 479 : cluster [DBG] pgmap v1006: 105 pgs: 4 creating+peering, 8 unknown, 93 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:51:52.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:52 smithi077 bash[80681]: cluster 2024-09-13T00:51:51.163999+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2024-09-13T00:51:52.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:52 smithi077 bash[83127]: cluster 2024-09-13T00:51:50.843763+0000 mgr.y (mgr.25304) 479 : cluster [DBG] pgmap v1006: 105 pgs: 4 creating+peering, 8 unknown, 93 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:51:52.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:52 smithi077 bash[83127]: cluster 2024-09-13T00:51:51.163999+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2024-09-13T00:51:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:52 smithi088 bash[68960]: cluster 2024-09-13T00:51:50.843763+0000 mgr.y (mgr.25304) 479 : cluster [DBG] pgmap v1006: 105 pgs: 4 creating+peering, 8 unknown, 93 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:51:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:52 smithi088 bash[68960]: cluster 2024-09-13T00:51:51.163999+0000 mon.a (mon.0) 1444 : cluster [DBG] osdmap e908: 8 total, 8 up, 8 in 2024-09-13T00:51:52.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:51:52 smithi088 bash[61791]: ts=2024-09-13T00:51:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:51:53.226 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 0~4194304 2024-09-13T00:51:53.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:53 smithi077 bash[80681]: cluster 2024-09-13T00:51:52.166710+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2024-09-13T00:51:53.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:53 smithi077 bash[83127]: cluster 2024-09-13T00:51:52.166710+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2024-09-13T00:51:53.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:53 smithi088 bash[68960]: cluster 2024-09-13T00:51:52.166710+0000 mon.a (mon.0) 1445 : cluster [DBG] osdmap e909: 8 total, 8 up, 8 in 2024-09-13T00:51:54.241 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 0~4194304 2024-09-13T00:51:54.244 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/1.DiffIterateDiscard (3028 ms) 2024-09-13T00:51:54.244 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStress 2024-09-13T00:51:54.244 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:51:54.416 INFO:tasks.workunit.client.0.smithi077.stdout: i=0 exists [75754778~2589703,104476321~6365436,127872446~3482191,144833030~7728519,301385457~5600289,312175522~5493439,343776091~3670848,363440567~2919313,372931578~7300298] wrote [75754778~2589703,104476321~6365436,127872446~3482191,144833030~7728519,301385457~5600289,312175522~5493439,343776091~3670848,363440567~2919313,372931578~7300298] 2024-09-13T00:51:54.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:54 smithi077 bash[80681]: cluster 2024-09-13T00:51:52.844537+0000 mgr.y (mgr.25304) 480 : cluster [DBG] pgmap v1009: 105 pgs: 4 creating+peering, 101 active+clean; 4.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.0 KiB/s rd, 73 KiB/s wr, 11 op/s 2024-09-13T00:51:54.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:54 smithi077 bash[80681]: cluster 2024-09-13T00:51:53.193955+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2024-09-13T00:51:54.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:54 smithi077 bash[83127]: cluster 2024-09-13T00:51:52.844537+0000 mgr.y (mgr.25304) 480 : cluster [DBG] pgmap v1009: 105 pgs: 4 creating+peering, 101 active+clean; 4.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.0 KiB/s rd, 73 KiB/s wr, 11 op/s 2024-09-13T00:51:54.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:54 smithi077 bash[83127]: cluster 2024-09-13T00:51:53.193955+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2024-09-13T00:51:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:54 smithi088 bash[68960]: cluster 2024-09-13T00:51:52.844537+0000 mgr.y (mgr.25304) 480 : cluster [DBG] pgmap v1009: 105 pgs: 4 creating+peering, 101 active+clean; 4.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.0 KiB/s rd, 73 KiB/s wr, 11 op/s 2024-09-13T00:51:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:54 smithi088 bash[68960]: cluster 2024-09-13T00:51:53.193955+0000 mon.a (mon.0) 1446 : cluster [DBG] osdmap e910: 8 total, 8 up, 8 in 2024-09-13T00:51:55.330 INFO:tasks.workunit.client.0.smithi077.stdout: i=1 exists [22006071~2614237,72775708~504908,75754778~2589703,104476321~6365436,127872446~3482191,144833030~7728519,164648423~5054609,211264662~4855495,259003838~4746120,283505360~7307345,301385457~5600289,312175522~5493439,343776091~3670848,363440567~2919313,369422624~10809252,381607704~7260759] wrote [22006071~2614237,72775708~504908,164648423~5054609,211264662~4855495,259003838~4746120,283505360~7307345,369422624~7791695,381607704~7260759] 2024-09-13T00:51:55.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:55 smithi077 bash[80681]: cluster 2024-09-13T00:51:54.202221+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2024-09-13T00:51:55.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:55 smithi077 bash[83127]: cluster 2024-09-13T00:51:54.202221+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2024-09-13T00:51:55.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:55 smithi088 bash[68960]: cluster 2024-09-13T00:51:54.202221+0000 mon.a (mon.0) 1447 : cluster [DBG] osdmap e911: 8 total, 8 up, 8 in 2024-09-13T00:51:56.250 INFO:tasks.workunit.client.0.smithi077.stdout: i=2 exists [22006071~2614237,33852135~6525373,72775708~504908,74957700~3386781,104476321~6365436,125096015~6258622,137036338~143705,144833030~7728519,164648423~5054609,173151540~5289128,193507257~10670905,211264662~4855495,242182386~2258254,259003838~4746120,283505360~7834472,301385457~5600289,312175522~5493439,343776091~3670848,363440567~2919313,369422624~11915112,381607704~7260759] wrote [33852135~6525373,74957700~2214767,125096015~3376369,137036338~143705,173151540~5289128,193507257~10670905,242182386~2258254,284589890~6749942,373925353~7412383] 2024-09-13T00:51:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:56 smithi077 bash[80681]: cluster 2024-09-13T00:51:54.845589+0000 mgr.y (mgr.25304) 481 : cluster [DBG] pgmap v1012: 105 pgs: 105 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 2.2 MiB/s wr, 135 op/s 2024-09-13T00:51:56.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:56 smithi077 bash[80681]: cluster 2024-09-13T00:51:55.211620+0000 mon.a (mon.0) 1448 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2024-09-13T00:51:56.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:56 smithi077 bash[80681]: cluster 2024-09-13T00:51:56.107419+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2024-09-13T00:51:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:56 smithi077 bash[83127]: cluster 2024-09-13T00:51:54.845589+0000 mgr.y (mgr.25304) 481 : cluster [DBG] pgmap v1012: 105 pgs: 105 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 2.2 MiB/s wr, 135 op/s 2024-09-13T00:51:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:56 smithi077 bash[83127]: cluster 2024-09-13T00:51:55.211620+0000 mon.a (mon.0) 1448 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2024-09-13T00:51:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:56 smithi077 bash[83127]: cluster 2024-09-13T00:51:56.107419+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2024-09-13T00:51:56.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:56 smithi088 bash[68960]: cluster 2024-09-13T00:51:54.845589+0000 mgr.y (mgr.25304) 481 : cluster [DBG] pgmap v1012: 105 pgs: 105 active+clean; 31 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 2.2 MiB/s wr, 135 op/s 2024-09-13T00:51:56.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:56 smithi088 bash[68960]: cluster 2024-09-13T00:51:55.211620+0000 mon.a (mon.0) 1448 : cluster [DBG] osdmap e912: 8 total, 8 up, 8 in 2024-09-13T00:51:56.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:56 smithi088 bash[68960]: cluster 2024-09-13T00:51:56.107419+0000 mon.a (mon.0) 1449 : cluster [DBG] osdmap e913: 8 total, 8 up, 8 in 2024-09-13T00:51:57.314 INFO:tasks.workunit.client.0.smithi077.stdout: i=3 exists [3969734~3927629,22006071~2614237,33852135~6525373,43831515~2135436,72109633~293889,72775708~504908,74957700~3386781,90734599~4075190,104476321~6365436,125096015~6258622,137036338~143705,144833030~8420395,164648423~5054609,173151540~5289128,181213669~2502623,193507257~10670905,204855433~3666654,211042210~5077947,216507034~130892,242182386~2258254,259003838~4746120,283505360~7834472,293494371~5053870,301385457~5600289,312175522~5493439,343776091~3670848,363440567~2919313,369422624~11915112,381607704~7260759] wrote [3969734~3927629,43831515~2135436,72109633~293889,90734599~4075190,145632680~7620745,181213669~2502623,204855433~3666654,211042210~1519031,216507034~130892,293494371~5053870] 2024-09-13T00:51:58.332 INFO:tasks.workunit.client.0.smithi077.stdout: i=4 exists [3969734~3927629,16224239~8396069,33852135~6525373,43831515~2135436,54314532~533177,72109633~293889,72775708~504908,74957700~3386781,90734599~4075190,101643649~1495885,104430048~6411709,125096015~6258622,137036338~143705,144833030~13150471,164648423~5054609,173151540~5289128,181213669~2502623,193507257~10670905,204855433~3666654,211042210~5077947,216507034~130892,242182386~2258254,259003838~4746120,283505360~7834472,292652290~5895951,301385457~5600289,312175522~5493439,343776091~3670848,350419784~2330391,363440567~2919313,369422624~11915112,381607704~7260759] wrote [16224239~5867393,54314532~533177,101643649~1495885,104430048~580975,150225682~7757819,198021651~5352202,292652290~4461555,350419784~2330391] 2024-09-13T00:51:58.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:58 smithi077 bash[80681]: cluster 2024-09-13T00:51:56.846393+0000 mgr.y (mgr.25304) 482 : cluster [DBG] pgmap v1015: 105 pgs: 105 active+clean; 65 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 98 KiB/s rd, 9.4 MiB/s wr, 161 op/s 2024-09-13T00:51:58.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:58 smithi077 bash[80681]: cluster 2024-09-13T00:51:57.233406+0000 mon.a (mon.0) 1450 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2024-09-13T00:51:58.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:58 smithi077 bash[83127]: cluster 2024-09-13T00:51:56.846393+0000 mgr.y (mgr.25304) 482 : cluster [DBG] pgmap v1015: 105 pgs: 105 active+clean; 65 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 98 KiB/s rd, 9.4 MiB/s wr, 161 op/s 2024-09-13T00:51:58.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:58 smithi077 bash[83127]: cluster 2024-09-13T00:51:57.233406+0000 mon.a (mon.0) 1450 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2024-09-13T00:51:58.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:58 smithi088 bash[68960]: cluster 2024-09-13T00:51:56.846393+0000 mgr.y (mgr.25304) 482 : cluster [DBG] pgmap v1015: 105 pgs: 105 active+clean; 65 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 98 KiB/s rd, 9.4 MiB/s wr, 161 op/s 2024-09-13T00:51:58.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:58 smithi088 bash[68960]: cluster 2024-09-13T00:51:57.233406+0000 mon.a (mon.0) 1450 : cluster [DBG] osdmap e914: 8 total, 8 up, 8 in 2024-09-13T00:51:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:51:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:51:58] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.43.0" 2024-09-13T00:51:59.330 INFO:tasks.workunit.client.0.smithi077.stdout: i=5 exists [3969734~3927629,16224239~8396069,33852135~6525373,43831515~2135436,54314532~533177,58015843~3488872,72109633~293889,72775708~504908,74920708~5221400,90734599~4075190,98349258~4790276,104430048~6411709,119324709~620408,125096015~6258622,137036338~143705,140065713~360686,144833030~13150471,164648423~5054609,173151540~7588790,181213669~2502623,193507257~10670905,204855433~3666654,211042210~5077947,216507034~130892,242182386~2258254,259003838~4746120,263869247~4645636,271133578~611740,283505360~7834472,292652290~5895951,301385457~5600289,312175522~5493439,343776091~3670848,350419784~2330391,363440567~2919313,369422624~11915112,381607704~7260759,398874397~4477166] wrote [58015843~3488872,74920708~5221400,98349258~3646830,119324709~620408,140065713~360686,150615170~4703519,176536667~4203663,263869247~4645636,271133578~611740,398874397~4477166] 2024-09-13T00:51:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:59 smithi077 bash[80681]: cluster 2024-09-13T00:51:58.236142+0000 mon.a (mon.0) 1451 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2024-09-13T00:51:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:59 smithi077 bash[80681]: audit 2024-09-13T00:51:58.427362+0000 mon.a (mon.0) 1452 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:51:59 smithi077 bash[80681]: audit 2024-09-13T00:51:58.763286+0000 mon.a (mon.0) 1453 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:59.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:59 smithi077 bash[83127]: cluster 2024-09-13T00:51:58.236142+0000 mon.a (mon.0) 1451 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2024-09-13T00:51:59.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:59 smithi077 bash[83127]: audit 2024-09-13T00:51:58.427362+0000 mon.a (mon.0) 1452 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:59.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:51:59 smithi077 bash[83127]: audit 2024-09-13T00:51:58.763286+0000 mon.a (mon.0) 1453 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:51:59.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:59 smithi088 bash[68960]: cluster 2024-09-13T00:51:58.236142+0000 mon.a (mon.0) 1451 : cluster [DBG] osdmap e915: 8 total, 8 up, 8 in 2024-09-13T00:51:59.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:59 smithi088 bash[68960]: audit 2024-09-13T00:51:58.427362+0000 mon.a (mon.0) 1452 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:51:59.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:51:59 smithi088 bash[68960]: audit 2024-09-13T00:51:58.763286+0000 mon.a (mon.0) 1453 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:52:00.365 INFO:tasks.workunit.client.0.smithi077.stdout: i=6 exists [3969734~3927629,16224239~8396069,33852135~6525373,43831515~2135436,50976621~994627,54314532~533177,56159548~5345167,72109633~293889,72775708~504908,74920708~5221400,81587013~5638716,90734599~4075190,98349258~4790276,104430048~6411709,119324709~620408,125096015~6258622,137036338~143705,140065713~360686,144833030~13150471,161136500~8566532,173151540~7588790,181213669~2502623,185244425~6473401,193507257~10670905,204855433~3666654,211042210~5077947,216507034~130892,242182386~2258254,259003838~4746120,263869247~4645636,271133578~611740,283505360~7834472,292652290~5895951,300984380~6001366,312175522~11515289,343776091~3670848,350419784~2330391,363440567~3014129,369422624~11915112,381607704~7260759,398874397~4477166] wrote [50976621~994627,56159548~2805633,81587013~5638716,126522315~212618,161136500~6166731,185244425~6473401,300984380~852793,317442515~6248296,364576907~1877789,372315945~707832] 2024-09-13T00:52:00.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:00 smithi077 bash[80681]: cluster 2024-09-13T00:51:58.847165+0000 mgr.y (mgr.25304) 483 : cluster [DBG] pgmap v1018: 105 pgs: 105 active+clean; 149 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 24 MiB/s wr, 45 op/s 2024-09-13T00:52:00.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:00 smithi077 bash[80681]: cluster 2024-09-13T00:51:59.250471+0000 mon.a (mon.0) 1454 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2024-09-13T00:52:00.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:00 smithi077 bash[83127]: cluster 2024-09-13T00:51:58.847165+0000 mgr.y (mgr.25304) 483 : cluster [DBG] pgmap v1018: 105 pgs: 105 active+clean; 149 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 24 MiB/s wr, 45 op/s 2024-09-13T00:52:00.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:00 smithi077 bash[83127]: cluster 2024-09-13T00:51:59.250471+0000 mon.a (mon.0) 1454 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2024-09-13T00:52:00.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:00 smithi088 bash[68960]: cluster 2024-09-13T00:51:58.847165+0000 mgr.y (mgr.25304) 483 : cluster [DBG] pgmap v1018: 105 pgs: 105 active+clean; 149 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 24 MiB/s wr, 45 op/s 2024-09-13T00:52:00.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:00 smithi088 bash[68960]: cluster 2024-09-13T00:51:59.250471+0000 mon.a (mon.0) 1454 : cluster [DBG] osdmap e916: 8 total, 8 up, 8 in 2024-09-13T00:52:01.341 INFO:tasks.workunit.client.0.smithi077.stdout: i=7 exists [858402~194509,3969734~3927629,16224239~8396069,33852135~6525373,43831515~2135436,50976621~994627,54314532~533177,56159548~5345167,72109633~293889,72775708~504908,74920708~5221400,81587013~6007316,90734599~4075190,98349258~4790276,104430048~6411709,119324709~620408,125096015~6258622,136962206~217837,140065713~360686,144833030~13150471,159325506~488314,161136500~8566532,173151540~7588790,181213669~2502623,185244425~6473401,193507257~10670905,204855433~6108807,211042210~5077947,216507034~130892,242182386~2258254,259003838~4746120,263869247~4645636,271133578~611740,283505360~7834472,292652290~5895951,300984380~7676176,312175522~11515289,335801072~6417381,343776091~3670848,350419784~2330391,356814679~530870,363440567~3014129,369422624~11915112,381607704~7260759,398874397~4477166] wrote [858402~194509,20005796~3399018,85337367~2256962,136962206~115299,159325506~488314,200326708~3430438,205724757~5239483,303996649~4663907,335801072~6417381,356814679~530870] 2024-09-13T00:52:01.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:01 smithi077 bash[80681]: cluster 2024-09-13T00:52:00.274694+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2024-09-13T00:52:01.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:01 smithi077 bash[83127]: cluster 2024-09-13T00:52:00.274694+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2024-09-13T00:52:01.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:01 smithi088 bash[68960]: cluster 2024-09-13T00:52:00.274694+0000 mon.a (mon.0) 1455 : cluster [DBG] osdmap e917: 8 total, 8 up, 8 in 2024-09-13T00:52:02.413 INFO:tasks.workunit.client.0.smithi077.stdout: i=8 exists [858402~194509,3969734~3927629,16224239~8396069,33852135~6525373,43831515~2135436,50976621~994627,54314532~533177,56159548~5345167,67029445~945564,72109633~293889,72775708~504908,74920708~5221400,81587013~6007316,90734599~4075190,98349258~4790276,104430048~6411709,119324709~620408,125096015~6258622,136962206~217837,140065713~360686,144833030~13150471,159325506~488314,161136500~8566532,173151540~7588790,181213669~2502623,185244425~6473401,193507257~10670905,204855433~6108807,211042210~5077947,216507034~130892,242182386~2258254,259003838~11707112,271133578~611740,283505360~7834472,291890452~6736309,300984380~7676176,312175522~11515289,332737009~9481444,343776091~3670848,350419784~2330391,356814679~530870,363440567~3014129,369422624~11915112,381607704~7260759,398874397~9565792] wrote [23726649~483384,67029445~945564,83129077~726425,99213419~2123967,261765622~8945328,283998466~5769781,291890452~6736309,332737009~5603231,401310152~7130037] 2024-09-13T00:52:02.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:02 smithi077 bash[80681]: cluster 2024-09-13T00:52:00.847822+0000 mgr.y (mgr.25304) 484 : cluster [DBG] pgmap v1021: 105 pgs: 105 active+clean; 250 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 156 KiB/s rd, 36 MiB/s wr, 257 op/s 2024-09-13T00:52:02.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:02 smithi077 bash[80681]: cluster 2024-09-13T00:52:01.284607+0000 mon.a (mon.0) 1456 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2024-09-13T00:52:02.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:02 smithi077 bash[83127]: cluster 2024-09-13T00:52:00.847822+0000 mgr.y (mgr.25304) 484 : cluster [DBG] pgmap v1021: 105 pgs: 105 active+clean; 250 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 156 KiB/s rd, 36 MiB/s wr, 257 op/s 2024-09-13T00:52:02.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:02 smithi077 bash[83127]: cluster 2024-09-13T00:52:01.284607+0000 mon.a (mon.0) 1456 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2024-09-13T00:52:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:02 smithi088 bash[68960]: cluster 2024-09-13T00:52:00.847822+0000 mgr.y (mgr.25304) 484 : cluster [DBG] pgmap v1021: 105 pgs: 105 active+clean; 250 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 156 KiB/s rd, 36 MiB/s wr, 257 op/s 2024-09-13T00:52:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:02 smithi088 bash[68960]: cluster 2024-09-13T00:52:01.284607+0000 mon.a (mon.0) 1456 : cluster [DBG] osdmap e918: 8 total, 8 up, 8 in 2024-09-13T00:52:02.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:52:02 smithi088 bash[61791]: ts=2024-09-13T00:52:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:52:03.399 INFO:tasks.workunit.client.0.smithi077.stdout: i=9 exists [858402~194509,3969734~3927629,16224239~8396069,33852135~6525373,41105529~4861422,50976621~994627,54314532~533177,56159548~5345167,67029445~945564,72109633~293889,72775708~504908,74920708~12673621,90734599~4075190,98349258~4790276,104430048~6411709,119324709~620408,125096015~6258622,136962206~217837,140065713~360686,144833030~13150471,159325506~488314,161136500~8566532,173151540~7588790,181213669~2502623,185244425~6473401,193507257~17456983,211042210~5077947,216507034~130892,242182386~2258254,251475314~3737205,259003838~11707112,271133578~611740,272183506~1207498,283505360~15121401,300984380~7676176,312175522~11515289,332737009~9481444,343776091~3670848,350419784~2330391,356814679~530870,363440567~3014129,369422624~11915112,381607704~7260759,398874397~9565792] wrote [41105529~4304919,79911295~3882410,196266038~9128327,251475314~3737205,263142682~4268959,272183506~1207498,287078168~5403303,302977098~3648849] 2024-09-13T00:52:03.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:03 smithi088 bash[68960]: cluster 2024-09-13T00:52:02.291887+0000 mon.a (mon.0) 1457 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2024-09-13T00:52:03.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:03 smithi077 bash[80681]: cluster 2024-09-13T00:52:02.291887+0000 mon.a (mon.0) 1457 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2024-09-13T00:52:03.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:03 smithi077 bash[83127]: cluster 2024-09-13T00:52:02.291887+0000 mon.a (mon.0) 1457 : cluster [DBG] osdmap e919: 8 total, 8 up, 8 in 2024-09-13T00:52:04.413 INFO:tasks.workunit.client.0.smithi077.stdout: i=10 exists [858402~194509,3969734~3927629,16224239~8396069,33852135~6525373,41105529~4861422,50976621~994627,54314532~533177,56159548~5345167,67029445~945564,72109633~293889,72775708~504908,74920708~12673621,90734599~4075190,95787202~7352332,104430048~6411709,119324709~620408,125096015~6258622,136962206~217837,140065713~360686,144833030~24870002,169876275~13840017,185244425~6473401,193507257~17456983,211042210~5077947,216507034~130892,242182386~2258254,249190421~6022098,259003838~11707112,271133578~611740,272183506~7065384,283505360~15121401,300984380~7676176,312175522~11515289,332737009~9481444,343776091~3670848,350419784~2330391,356814679~530870,363440567~3014129,369422624~11915112,381607704~7517863,398874397~9565792] wrote [95787202~5943450,157532294~8067551,169876275~6336921,180270980~3294823,202546355~7400062,214250537~299603,249190421~3929733,272996731~6252159,318579772~2517412,387058799~2066768] 2024-09-13T00:52:04.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:04 smithi088 bash[68960]: cluster 2024-09-13T00:52:02.848499+0000 mgr.y (mgr.25304) 485 : cluster [DBG] pgmap v1024: 105 pgs: 105 active+clean; 297 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 157 KiB/s rd, 28 MiB/s wr, 255 op/s 2024-09-13T00:52:04.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:04 smithi088 bash[68960]: cluster 2024-09-13T00:52:03.303793+0000 mon.a (mon.0) 1458 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2024-09-13T00:52:04.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:04 smithi077 bash[80681]: cluster 2024-09-13T00:52:02.848499+0000 mgr.y (mgr.25304) 485 : cluster [DBG] pgmap v1024: 105 pgs: 105 active+clean; 297 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 157 KiB/s rd, 28 MiB/s wr, 255 op/s 2024-09-13T00:52:04.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:04 smithi077 bash[80681]: cluster 2024-09-13T00:52:03.303793+0000 mon.a (mon.0) 1458 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2024-09-13T00:52:04.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:04 smithi077 bash[83127]: cluster 2024-09-13T00:52:02.848499+0000 mgr.y (mgr.25304) 485 : cluster [DBG] pgmap v1024: 105 pgs: 105 active+clean; 297 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 157 KiB/s rd, 28 MiB/s wr, 255 op/s 2024-09-13T00:52:04.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:04 smithi077 bash[83127]: cluster 2024-09-13T00:52:03.303793+0000 mon.a (mon.0) 1458 : cluster [DBG] osdmap e920: 8 total, 8 up, 8 in 2024-09-13T00:52:05.421 INFO:tasks.workunit.client.0.smithi077.stdout: i=11 exists [858402~194509,3969734~3927629,16137726~8482582,31730480~14236471,50976621~994627,54314532~533177,56159548~5345167,67029445~945564,72109633~293889,72775708~504908,74920708~12673621,90734599~4075190,95787202~7352332,104430048~6411709,119324709~620408,125096015~8636032,136962206~217837,140065713~360686,144833030~24870002,169876275~21841551,193507257~17456983,211042210~5077947,216507034~130892,242182386~2258254,248628004~6584515,259003838~11707112,271133578~611740,272183506~7065384,283505360~15121401,300984380~7676176,312175522~11515289,327861456~14356997,343776091~3670848,350419784~3644731,356814679~530870,363440567~3014129,369422624~11915112,381607704~7517863,396684144~886108,398874397~9565792] wrote [16137726~3066269,31730480~3753076,36404278~6116834,127005121~6726926,183277725~2571140,248628004~1673258,327861456~7448904,351214947~2849568,381841366~2251939,396684144~886108] 2024-09-13T00:52:05.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:05 smithi088 bash[68960]: cluster 2024-09-13T00:52:04.314426+0000 mon.a (mon.0) 1459 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2024-09-13T00:52:05.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:05 smithi088 bash[68960]: audit 2024-09-13T00:52:04.445298+0000 mon.a (mon.0) 1460 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:52:05.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:05 smithi077 bash[80681]: cluster 2024-09-13T00:52:04.314426+0000 mon.a (mon.0) 1459 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2024-09-13T00:52:05.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:05 smithi077 bash[80681]: audit 2024-09-13T00:52:04.445298+0000 mon.a (mon.0) 1460 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:52:05.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:05 smithi077 bash[83127]: cluster 2024-09-13T00:52:04.314426+0000 mon.a (mon.0) 1459 : cluster [DBG] osdmap e921: 8 total, 8 up, 8 in 2024-09-13T00:52:05.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:05 smithi077 bash[83127]: audit 2024-09-13T00:52:04.445298+0000 mon.a (mon.0) 1460 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:52:06.433 INFO:tasks.workunit.client.0.smithi077.stdout: i=12 exists [858402~194509,3969734~3927629,16137726~8482582,31730480~14236471,46365060~5606188,54314532~533177,56159548~5345167,67029445~945564,72109633~293889,72775708~504908,74772724~15169888,90734599~4075190,95787202~7352332,104430048~6411709,119324709~620408,125096015~8636032,136962206~217837,140065713~360686,144833030~24870002,169876275~21841551,193507257~17456983,211042210~5077947,216507034~130892,242182386~2258254,248628004~6584515,259003838~11707112,271133578~611740,272183506~10061253,283505360~15121401,300984380~22706431,327861456~14356997,343776091~3670848,350419784~5800465,356814679~530870,363440567~3014129,369422624~11915112,381607704~7517863,396684144~886108,398874397~9565792] wrote [46365060~4622165,74772724~8088588,86638224~3304388,173129534~4282946,183091616~3528016,187607573~2259663,274984613~7260146,307993644~6593992,353324638~2895611] 2024-09-13T00:52:06.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:06 smithi088 bash[68960]: cluster 2024-09-13T00:52:04.849420+0000 mgr.y (mgr.25304) 486 : cluster [DBG] pgmap v1027: 105 pgs: 105 active+clean; 421 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 39 MiB/s wr, 306 op/s 2024-09-13T00:52:06.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:06 smithi088 bash[68960]: cluster 2024-09-13T00:52:05.324855+0000 mon.a (mon.0) 1461 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2024-09-13T00:52:06.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:06 smithi077 bash[83127]: cluster 2024-09-13T00:52:04.849420+0000 mgr.y (mgr.25304) 486 : cluster [DBG] pgmap v1027: 105 pgs: 105 active+clean; 421 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 39 MiB/s wr, 306 op/s 2024-09-13T00:52:06.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:06 smithi077 bash[83127]: cluster 2024-09-13T00:52:05.324855+0000 mon.a (mon.0) 1461 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2024-09-13T00:52:06.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:06 smithi077 bash[80681]: cluster 2024-09-13T00:52:04.849420+0000 mgr.y (mgr.25304) 486 : cluster [DBG] pgmap v1027: 105 pgs: 105 active+clean; 421 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 39 MiB/s wr, 306 op/s 2024-09-13T00:52:06.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:06 smithi077 bash[80681]: cluster 2024-09-13T00:52:05.324855+0000 mon.a (mon.0) 1461 : cluster [DBG] osdmap e922: 8 total, 8 up, 8 in 2024-09-13T00:52:07.456 INFO:tasks.workunit.client.0.smithi077.stdout: i=13 exists [858402~194509,3969734~3927629,13791976~16343562,31730480~14236471,46365060~5606188,54314532~533177,56159548~5345167,61559946~7538596,72109633~293889,72775708~504908,74772724~15169888,90734599~4075190,95787202~7718722,104430048~6411709,119324709~620408,125096015~8636032,136962206~217837,139815791~2100307,144833030~24870002,169876275~21841551,193507257~17456983,211042210~5077947,216507034~130892,242182386~2258254,248628004~6584515,259003838~11707112,271133578~611740,272183506~26443255,300984380~22706431,324905495~17312958,343776091~3670848,350419784~5800465,356814679~530870,363440567~3014129,369422624~11915112,381607704~7517863,396684144~886108,398874397~9565792] wrote [13791976~8178393,22937421~7198117,61559946~7538596,88365823~73074,96178889~7327035,104880712~2288324,139815791~2100307,280170704~4043013,324905495~7357831,372603408~2515349] 2024-09-13T00:52:07.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:07 smithi088 bash[68960]: cluster 2024-09-13T00:52:06.325914+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2024-09-13T00:52:07.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:07 smithi077 bash[80681]: cluster 2024-09-13T00:52:06.325914+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2024-09-13T00:52:07.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:07 smithi077 bash[83127]: cluster 2024-09-13T00:52:06.325914+0000 mon.a (mon.0) 1462 : cluster [DBG] osdmap e923: 8 total, 8 up, 8 in 2024-09-13T00:52:08.480 INFO:tasks.workunit.client.0.smithi077.stdout: i=14 exists [858402~194509,3969734~3927629,13402144~38569104,54314532~533177,56159548~12938994,72109633~293889,72775708~504908,74772724~15169888,90734599~4075190,95787202~7718722,104430048~6411709,119324709~620408,125096015~8636032,136962206~217837,139815791~2100307,144833030~46884796,193507257~17456983,211042210~5077947,216507034~130892,242182386~2258254,248628004~6584515,259003838~11707112,271133578~611740,272183506~26443255,300984380~22706431,324905495~17312958,343776091~3670848,350419784~5800465,356814679~530870,363440567~3014129,369422624~11915112,381607704~7517863,396684144~886108,398874397~9565792] wrote [13402144~5702843,28876281~7462179,44659836~4078381,57762873~6626970,164308539~5836504,177730534~2979132,193574394~7022051,273377844~3421536] 2024-09-13T00:52:08.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:08 smithi088 bash[68960]: cluster 2024-09-13T00:52:06.850284+0000 mgr.y (mgr.25304) 487 : cluster [DBG] pgmap v1030: 105 pgs: 105 active+clean; 494 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 45 MiB/s wr, 310 op/s 2024-09-13T00:52:08.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:08 smithi088 bash[68960]: cluster 2024-09-13T00:52:07.342143+0000 mon.a (mon.0) 1463 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2024-09-13T00:52:08.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:52:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:52:08] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.43.0" 2024-09-13T00:52:08.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:08 smithi077 bash[80681]: cluster 2024-09-13T00:52:06.850284+0000 mgr.y (mgr.25304) 487 : cluster [DBG] pgmap v1030: 105 pgs: 105 active+clean; 494 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 45 MiB/s wr, 310 op/s 2024-09-13T00:52:08.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:08 smithi077 bash[80681]: cluster 2024-09-13T00:52:07.342143+0000 mon.a (mon.0) 1463 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2024-09-13T00:52:08.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:08 smithi077 bash[83127]: cluster 2024-09-13T00:52:06.850284+0000 mgr.y (mgr.25304) 487 : cluster [DBG] pgmap v1030: 105 pgs: 105 active+clean; 494 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 45 MiB/s wr, 310 op/s 2024-09-13T00:52:08.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:08 smithi077 bash[83127]: cluster 2024-09-13T00:52:07.342143+0000 mon.a (mon.0) 1463 : cluster [DBG] osdmap e924: 8 total, 8 up, 8 in 2024-09-13T00:52:09.414 INFO:tasks.workunit.client.0.smithi077.stdout: i=15 exists [858402~194509,3969734~3927629,13402144~38569104,54314532~533177,56159548~12938994,72109633~293889,72775708~504908,74772724~15169888,90734599~4075190,95787202~7718722,104430048~6411709,116916279~2168849,119324709~620408,125096015~8636032,136962206~217837,139815791~2100307,144833030~46884796,193507257~17456983,211042210~5077947,216507034~130892,242182386~2258254,248628004~6584515,259003838~11707112,271133578~611740,272183506~26443255,300984380~22706431,324905495~17312958,343776091~3670848,350419784~5800465,356814679~530870,363440567~3014129,369422624~20449815,393363125~103170,396684144~886108,398874397~9565792] wrote [116916279~2168849,168879432~6072873,184188764~1502822,187241080~1367948,264347475~1522138,304228751~3974693,353714660~1569760,378457094~7800418,387166665~2705774,393363125~103170] 2024-09-13T00:52:09.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:09 smithi077 bash[80681]: cluster 2024-09-13T00:52:08.354812+0000 mon.a (mon.0) 1464 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2024-09-13T00:52:09.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:09 smithi077 bash[83127]: cluster 2024-09-13T00:52:08.354812+0000 mon.a (mon.0) 1464 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2024-09-13T00:52:09.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:09 smithi088 bash[68960]: cluster 2024-09-13T00:52:08.354812+0000 mon.a (mon.0) 1464 : cluster [DBG] osdmap e925: 8 total, 8 up, 8 in 2024-09-13T00:52:10.451 INFO:tasks.workunit.client.0.smithi077.stdout: i=16 exists [858402~194509,3969734~3927629,13402144~38569104,54314532~533177,56159548~12938994,72109633~293889,72775708~504908,74772724~15169888,90734599~4075190,95787202~7718722,104430048~6411709,116916279~2168849,119216853~1612172,125096015~8636032,136962206~217837,139815791~2100307,144833030~46884796,193507257~17456983,211042210~5077947,216507034~130892,242182386~2258254,248628004~22082946,271133578~611740,272183506~26443255,300984380~22706431,324905495~22541444,350419784~5800465,356814679~530870,361977029~4477667,369422624~20449815,393363125~103170,396684144~886108,398874397~9565792] wrote [37989901~2725607,119216853~1612172,201653132~1258963,254922941~4435359,303633140~5118738,316406736~6245734,336907161~7660923,361977029~3558443] 2024-09-13T00:52:10.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:10 smithi077 bash[83127]: cluster 2024-09-13T00:52:08.851161+0000 mgr.y (mgr.25304) 488 : cluster [DBG] pgmap v1033: 105 pgs: 105 active+clean; 549 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 29 MiB/s wr, 16 op/s 2024-09-13T00:52:10.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:10 smithi077 bash[83127]: cluster 2024-09-13T00:52:09.358986+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2024-09-13T00:52:10.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:10 smithi077 bash[83127]: audit 2024-09-13T00:52:09.984592+0000 mon.a (mon.0) 1466 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:52:10.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:10 smithi077 bash[83127]: audit 2024-09-13T00:52:09.991530+0000 mon.a (mon.0) 1467 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:52:10.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:10 smithi077 bash[83127]: audit 2024-09-13T00:52:10.231742+0000 mon.a (mon.0) 1468 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:52:10.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:10 smithi077 bash[83127]: audit 2024-09-13T00:52:10.243337+0000 mon.a (mon.0) 1469 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:52:10.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:10 smithi077 bash[80681]: cluster 2024-09-13T00:52:08.851161+0000 mgr.y (mgr.25304) 488 : cluster [DBG] pgmap v1033: 105 pgs: 105 active+clean; 549 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 29 MiB/s wr, 16 op/s 2024-09-13T00:52:10.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:10 smithi077 bash[80681]: cluster 2024-09-13T00:52:09.358986+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2024-09-13T00:52:10.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:10 smithi077 bash[80681]: audit 2024-09-13T00:52:09.984592+0000 mon.a (mon.0) 1466 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:52:10.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:10 smithi077 bash[80681]: audit 2024-09-13T00:52:09.991530+0000 mon.a (mon.0) 1467 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:52:10.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:10 smithi077 bash[80681]: audit 2024-09-13T00:52:10.231742+0000 mon.a (mon.0) 1468 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:52:10.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:10 smithi077 bash[80681]: audit 2024-09-13T00:52:10.243337+0000 mon.a (mon.0) 1469 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:52:10.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:10 smithi088 bash[68960]: cluster 2024-09-13T00:52:08.851161+0000 mgr.y (mgr.25304) 488 : cluster [DBG] pgmap v1033: 105 pgs: 105 active+clean; 549 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1023 B/s rd, 29 MiB/s wr, 16 op/s 2024-09-13T00:52:10.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:10 smithi088 bash[68960]: cluster 2024-09-13T00:52:09.358986+0000 mon.a (mon.0) 1465 : cluster [DBG] osdmap e926: 8 total, 8 up, 8 in 2024-09-13T00:52:10.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:10 smithi088 bash[68960]: audit 2024-09-13T00:52:09.984592+0000 mon.a (mon.0) 1466 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:52:10.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:10 smithi088 bash[68960]: audit 2024-09-13T00:52:09.991530+0000 mon.a (mon.0) 1467 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:52:10.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:10 smithi088 bash[68960]: audit 2024-09-13T00:52:10.231742+0000 mon.a (mon.0) 1468 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:52:10.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:10 smithi088 bash[68960]: audit 2024-09-13T00:52:10.243337+0000 mon.a (mon.0) 1469 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:52:11.485 INFO:tasks.workunit.client.0.smithi077.stdout: i=17 exists [858402~194509,3969734~8481846,13402144~38569104,54314532~533177,56159548~12938994,72109633~293889,72775708~504908,74138618~15803994,90734599~4075190,95787202~7718722,104430048~6411709,114914535~4170593,119216853~1612172,123821269~9910778,136962206~217837,139815791~2786453,144833030~46884796,193507257~17456983,211042210~5077947,216507034~130892,242182386~2258254,248628004~22082946,271133578~611740,272183506~26443255,300984380~48417917,350419784~5800465,356814679~530870,359110778~7343918,369422624~23393742,393363125~103170,396684144~886108,398874397~9565792] wrote [5740297~6711283,57900766~4150902,74138618~968085,114914535~2319020,123821269~4323814,140420383~2181861,322850155~5730426,343448575~5953722,359110778~6711367,389060303~3756063] 2024-09-13T00:52:11.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:11 smithi077 bash[83127]: cluster 2024-09-13T00:52:10.372313+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2024-09-13T00:52:11.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:11 smithi077 bash[83127]: audit 2024-09-13T00:52:10.705876+0000 mon.a (mon.0) 1471 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:52:11.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:11 smithi077 bash[83127]: audit 2024-09-13T00:52:10.707441+0000 mon.a (mon.0) 1472 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:52:11.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:11 smithi077 bash[83127]: audit 2024-09-13T00:52:10.714848+0000 mon.a (mon.0) 1473 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:52:11.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:11 smithi077 bash[80681]: cluster 2024-09-13T00:52:10.372313+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2024-09-13T00:52:11.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:11 smithi077 bash[80681]: audit 2024-09-13T00:52:10.705876+0000 mon.a (mon.0) 1471 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:52:11.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:11 smithi077 bash[80681]: audit 2024-09-13T00:52:10.707441+0000 mon.a (mon.0) 1472 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:52:11.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:11 smithi077 bash[80681]: audit 2024-09-13T00:52:10.714848+0000 mon.a (mon.0) 1473 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:52:11.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:11 smithi088 bash[68960]: cluster 2024-09-13T00:52:10.372313+0000 mon.a (mon.0) 1470 : cluster [DBG] osdmap e927: 8 total, 8 up, 8 in 2024-09-13T00:52:11.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:11 smithi088 bash[68960]: audit 2024-09-13T00:52:10.705876+0000 mon.a (mon.0) 1471 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:52:11.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:11 smithi088 bash[68960]: audit 2024-09-13T00:52:10.707441+0000 mon.a (mon.0) 1472 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:52:11.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:11 smithi088 bash[68960]: audit 2024-09-13T00:52:10.714848+0000 mon.a (mon.0) 1473 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:52:12.519 INFO:tasks.workunit.client.0.smithi077.stdout: i=18 exists [858402~194509,3969734~8481846,13402144~41565197,56159548~12938994,72109633~293889,72775708~504908,74138618~15803994,90734599~4075190,95787202~7718722,104430048~6411709,114914535~7684362,123821269~9910778,136962206~217837,139815791~2786453,144833030~46884796,193507257~17456983,211042210~5077947,216507034~130892,242122446~28588504,271133578~611740,272183506~26443255,300984380~48417917,350419784~5800465,356814679~530870,359110778~9510139,368897625~243553,369422624~23393742,393363125~103170,396684144~886108,398874397~9565792] wrote [15946371~5437678,34733337~6500714,49653387~5313954,84378455~1751969,116159330~6439567,242122446~7545670,360386004~8234913,368897625~243553] 2024-09-13T00:52:12.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:12 smithi088 bash[68960]: cluster 2024-09-13T00:52:10.851830+0000 mgr.y (mgr.25304) 489 : cluster [DBG] pgmap v1036: 105 pgs: 105 active+clean; 624 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 222 KiB/s rd, 32 MiB/s wr, 343 op/s 2024-09-13T00:52:12.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:12 smithi088 bash[68960]: cluster 2024-09-13T00:52:11.389721+0000 mon.a (mon.0) 1474 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2024-09-13T00:52:12.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:52:12 smithi088 bash[61791]: ts=2024-09-13T00:52:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:52:12.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:12 smithi077 bash[80681]: cluster 2024-09-13T00:52:10.851830+0000 mgr.y (mgr.25304) 489 : cluster [DBG] pgmap v1036: 105 pgs: 105 active+clean; 624 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 222 KiB/s rd, 32 MiB/s wr, 343 op/s 2024-09-13T00:52:12.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:12 smithi077 bash[80681]: cluster 2024-09-13T00:52:11.389721+0000 mon.a (mon.0) 1474 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2024-09-13T00:52:12.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:12 smithi077 bash[83127]: cluster 2024-09-13T00:52:10.851830+0000 mgr.y (mgr.25304) 489 : cluster [DBG] pgmap v1036: 105 pgs: 105 active+clean; 624 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 222 KiB/s rd, 32 MiB/s wr, 343 op/s 2024-09-13T00:52:12.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:12 smithi077 bash[83127]: cluster 2024-09-13T00:52:11.389721+0000 mon.a (mon.0) 1474 : cluster [DBG] osdmap e928: 8 total, 8 up, 8 in 2024-09-13T00:52:13.540 INFO:tasks.workunit.client.0.smithi077.stdout: i=19 exists [858402~194509,3969734~8481846,13402144~41565197,56159548~12938994,72109633~293889,72775708~504908,74138618~15803994,90734599~4075190,95787202~7718722,104430048~6411709,114914535~8412410,123821269~9910778,136962206~217837,139815791~2786453,144833030~46884796,193507257~17456983,211042210~5077947,216507034~130892,223278528~7932841,242122446~28588504,271133578~611740,272183506~26443255,300984380~48417917,350419784~5800465,356814679~530870,359110778~9510139,368897625~243553,369422624~23393742,393363125~103170,396684144~886108,398874397~9565792] wrote [14602907~7478289,40679450~5794477,104959110~5720660,115192277~8134668,185698385~2987664,223278528~7932841,267298594~1003308,278352428~4518624,301153553~8037559] 2024-09-13T00:52:13.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:13 smithi077 bash[80681]: cluster 2024-09-13T00:52:12.392725+0000 mon.a (mon.0) 1475 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2024-09-13T00:52:13.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:13 smithi077 bash[83127]: cluster 2024-09-13T00:52:12.392725+0000 mon.a (mon.0) 1475 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2024-09-13T00:52:13.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:13 smithi088 bash[68960]: cluster 2024-09-13T00:52:12.392725+0000 mon.a (mon.0) 1475 : cluster [DBG] osdmap e929: 8 total, 8 up, 8 in 2024-09-13T00:52:14.435 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 1 diff [20971520~4194304,71303168~4194304,163577856~8388608,209715200~8388608,255852544~8388608,281018368~8388608,369098752~16777216] 2024-09-13T00:52:14.435 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [20971520~4194304,71303168~4194304,163577856~8388608,209715200~8388608,255852544~8388608,281018368~8388608,369098752~16777216] 2024-09-13T00:52:14.463 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [20971520~4194304,71303168~4194304,163577856~8388608,209715200~8388608,255852544~8388608,281018368~12582912,369098752~20971520] 2024-09-13T00:52:14.463 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 2 diff [20971520~4194304,33554432~8388608,71303168~4194304,121634816~4194304,134217728~4194304,163577856~16777216,192937984~12582912,209715200~8388608,239075328~4194304,255852544~8388608,281018368~8388608,369098752~16777216] 2024-09-13T00:52:14.463 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [20971520~4194304,33554432~8388608,71303168~4194304,121634816~4194304,134217728~4194304,163577856~16777216,192937984~12582912,209715200~8388608,239075328~4194304,255852544~8388608,281018368~8388608,369098752~16777216] 2024-09-13T00:52:14.471 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [20971520~4194304,33554432~8388608,71303168~8388608,121634816~8388608,134217728~4194304,163577856~16777216,192937984~12582912,209715200~8388608,239075328~8388608,255852544~8388608,281018368~12582912,369098752~20971520] 2024-09-13T00:52:14.471 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 3 diff [0~4194304,20971520~4194304,33554432~12582912,71303168~4194304,88080384~4194304,121634816~4194304,134217728~4194304,142606336~8388608,163577856~20971520,192937984~12582912,209715200~8388608,239075328~4194304,255852544~8388608,281018368~16777216,369098752~16777216] 2024-09-13T00:52:14.471 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,20971520~4194304,33554432~12582912,71303168~4194304,88080384~4194304,121634816~4194304,134217728~4194304,142606336~8388608,163577856~20971520,192937984~12582912,209715200~8388608,239075328~4194304,255852544~8388608,281018368~16777216,369098752~16777216] 2024-09-13T00:52:14.477 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,20971520~4194304,33554432~12582912,71303168~8388608,88080384~8388608,121634816~8388608,134217728~4194304,142606336~12582912,163577856~20971520,192937984~25165824,239075328~8388608,255852544~8388608,281018368~20971520,369098752~20971520] 2024-09-13T00:52:14.477 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 4 diff [0~4194304,12582912~12582912,33554432~12582912,50331648~4194304,71303168~4194304,88080384~4194304,100663296~4194304,121634816~4194304,134217728~4194304,142606336~12582912,163577856~20971520,192937984~12582912,209715200~8388608,239075328~4194304,255852544~8388608,281018368~16777216,348127232~4194304,369098752~16777216] 2024-09-13T00:52:14.477 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,33554432~12582912,50331648~4194304,71303168~4194304,88080384~4194304,100663296~4194304,121634816~4194304,134217728~4194304,142606336~12582912,163577856~20971520,192937984~12582912,209715200~8388608,239075328~4194304,255852544~8388608,281018368~16777216,348127232~4194304,369098752~16777216] 2024-09-13T00:52:14.483 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,50331648~8388608,71303168~8388608,88080384~8388608,100663296~8388608,121634816~8388608,134217728~4194304,142606336~16777216,163577856~20971520,192937984~25165824,239075328~8388608,255852544~8388608,281018368~20971520,348127232~8388608,369098752~20971520] 2024-09-13T00:52:14.483 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 5 diff [0~4194304,12582912~12582912,33554432~12582912,50331648~8388608,71303168~8388608,88080384~4194304,96468992~8388608,117440512~8388608,134217728~20971520,163577856~20971520,192937984~12582912,209715200~8388608,239075328~4194304,255852544~16777216,281018368~16777216,348127232~4194304,369098752~16777216,398458880~8388608] 2024-09-13T00:52:14.483 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,33554432~12582912,50331648~8388608,71303168~8388608,88080384~4194304,96468992~8388608,117440512~8388608,134217728~20971520,163577856~20971520,192937984~12582912,209715200~8388608,239075328~4194304,255852544~16777216,281018368~16777216,348127232~4194304,369098752~16777216,398458880~8388608] 2024-09-13T00:52:14.491 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,50331648~12582912,71303168~12582912,88080384~20971520,117440512~12582912,134217728~25165824,163577856~20971520,192937984~25165824,239075328~8388608,255852544~16777216,281018368~20971520,348127232~8388608,369098752~20971520,398458880~8388608] 2024-09-13T00:52:14.491 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 6 diff [0~4194304,12582912~12582912,33554432~12582912,50331648~12582912,71303168~20971520,96468992~8388608,117440512~8388608,134217728~20971520,159383552~46137344,209715200~8388608,239075328~4194304,255852544~16777216,281018368~20971520,314572800~8388608,348127232~4194304,360710144~4194304,369098752~16777216,398458880~8388608] 2024-09-13T00:52:14.491 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,33554432~12582912,50331648~12582912,71303168~20971520,96468992~8388608,117440512~8388608,134217728~20971520,159383552~46137344,209715200~8388608,239075328~4194304,255852544~16777216,281018368~20971520,314572800~8388608,348127232~4194304,360710144~4194304,369098752~16777216,398458880~8388608] 2024-09-13T00:52:14.499 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,50331648~12582912,71303168~37748736,117440512~12582912,134217728~83886080,239075328~8388608,255852544~16777216,281018368~20971520,314572800~12582912,348127232~8388608,360710144~29360128,398458880~8388608] 2024-09-13T00:52:14.500 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 7 diff [0~4194304,12582912~12582912,33554432~12582912,50331648~12582912,71303168~20971520,96468992~8388608,117440512~8388608,134217728~83886080,239075328~4194304,255852544~16777216,281018368~29360128,314572800~8388608,335544320~8388608,348127232~4194304,356515840~8388608,369098752~16777216,398458880~8388608] 2024-09-13T00:52:14.500 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,33554432~12582912,50331648~12582912,71303168~20971520,96468992~8388608,117440512~8388608,134217728~83886080,239075328~4194304,255852544~16777216,281018368~29360128,314572800~8388608,335544320~8388608,348127232~4194304,356515840~8388608,369098752~16777216,398458880~8388608] 2024-09-13T00:52:14.508 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,50331648~12582912,71303168~37748736,117440512~12582912,134217728~83886080,239075328~8388608,255852544~16777216,281018368~29360128,314572800~12582912,335544320~8388608,348127232~41943040,398458880~8388608] 2024-09-13T00:52:14.508 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 8 diff [0~4194304,12582912~12582912,33554432~12582912,50331648~16777216,71303168~20971520,96468992~8388608,117440512~8388608,134217728~83886080,239075328~4194304,255852544~16777216,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~16777216,398458880~12582912] 2024-09-13T00:52:14.508 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,33554432~12582912,50331648~16777216,71303168~20971520,96468992~8388608,117440512~8388608,134217728~83886080,239075328~4194304,255852544~16777216,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~16777216,398458880~12582912] 2024-09-13T00:52:14.518 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,50331648~58720256,117440512~12582912,134217728~83886080,239075328~8388608,255852544~16777216,281018368~29360128,314572800~12582912,331350016~12582912,348127232~41943040,398458880~12582912] 2024-09-13T00:52:14.518 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 9 diff [0~4194304,12582912~12582912,33554432~12582912,50331648~16777216,71303168~20971520,96468992~8388608,117440512~8388608,134217728~83886080,239075328~4194304,247463936~4194304,255852544~16777216,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~16777216,398458880~12582912] 2024-09-13T00:52:14.519 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,33554432~12582912,50331648~16777216,71303168~20971520,96468992~8388608,117440512~8388608,134217728~83886080,239075328~4194304,247463936~4194304,255852544~16777216,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~16777216,398458880~12582912] 2024-09-13T00:52:14.529 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,50331648~58720256,117440512~12582912,134217728~83886080,239075328~37748736,281018368~29360128,314572800~12582912,331350016~12582912,348127232~41943040,398458880~12582912] 2024-09-13T00:52:14.529 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 10 diff [0~4194304,12582912~12582912,33554432~12582912,50331648~16777216,71303168~33554432,117440512~8388608,134217728~83886080,239075328~4194304,247463936~29360128,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~16777216,398458880~12582912] 2024-09-13T00:52:14.529 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,33554432~12582912,50331648~16777216,71303168~33554432,117440512~8388608,134217728~83886080,239075328~4194304,247463936~29360128,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~16777216,398458880~12582912] 2024-09-13T00:52:14.541 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,50331648~58720256,117440512~12582912,134217728~83886080,239075328~71303168,314572800~12582912,331350016~12582912,348127232~41943040,398458880~12582912] 2024-09-13T00:52:14.541 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 11 diff [0~4194304,12582912~12582912,29360128~16777216,50331648~16777216,71303168~33554432,117440512~100663296,239075328~4194304,247463936~29360128,281018368~29360128,314572800~8388608,327155712~16777216,348127232~4194304,356515840~8388608,369098752~16777216,394264576~16777216] 2024-09-13T00:52:14.541 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,29360128~16777216,50331648~16777216,71303168~33554432,117440512~100663296,239075328~4194304,247463936~29360128,281018368~29360128,314572800~8388608,327155712~16777216,348127232~4194304,356515840~8388608,369098752~16777216,394264576~16777216] 2024-09-13T00:52:14.554 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,29360128~16777216,50331648~58720256,117440512~100663296,239075328~71303168,314572800~29360128,348127232~41943040,394264576~16777216] 2024-09-13T00:52:14.555 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 12 diff [0~4194304,12582912~12582912,29360128~37748736,71303168~33554432,117440512~100663296,239075328~4194304,247463936~75497472,327155712~16777216,348127232~16777216,369098752~16777216,394264576~16777216] 2024-09-13T00:52:14.555 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,29360128~37748736,71303168~33554432,117440512~100663296,239075328~4194304,247463936~75497472,327155712~16777216,348127232~16777216,369098752~16777216,394264576~16777216] 2024-09-13T00:52:14.568 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,29360128~79691776,117440512~100663296,239075328~104857600,348127232~41943040,394264576~16777216] 2024-09-13T00:52:14.568 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 13 diff [0~4194304,12582912~54525952,71303168~33554432,117440512~100663296,239075328~4194304,247463936~96468992,348127232~16777216,369098752~16777216,394264576~16777216] 2024-09-13T00:52:14.568 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~54525952,71303168~33554432,117440512~100663296,239075328~4194304,247463936~96468992,348127232~16777216,369098752~16777216,394264576~16777216] 2024-09-13T00:52:14.583 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~96468992,117440512~100663296,239075328~104857600,348127232~41943040,394264576~16777216] 2024-09-13T00:52:14.583 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 14 diff [0~4194304,12582912~92274688,117440512~71303168,192937984~25165824,239075328~4194304,247463936~96468992,348127232~16777216,369098752~16777216,394264576~16777216] 2024-09-13T00:52:14.583 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~92274688,117440512~71303168,192937984~25165824,239075328~4194304,247463936~96468992,348127232~16777216,369098752~16777216,394264576~16777216] 2024-09-13T00:52:14.600 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~96468992,117440512~100663296,239075328~104857600,348127232~41943040,394264576~16777216] 2024-09-13T00:52:14.600 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 15 diff [0~4194304,12582912~92274688,113246208~75497472,192937984~25165824,239075328~4194304,247463936~96468992,348127232~16777216,369098752~41943040] 2024-09-13T00:52:14.600 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~92274688,113246208~75497472,192937984~25165824,239075328~4194304,247463936~96468992,348127232~16777216,369098752~41943040] 2024-09-13T00:52:14.615 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~96468992,113246208~104857600,239075328~104857600,348127232~62914560] 2024-09-13T00:52:14.616 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 16 diff [0~4194304,12582912~92274688,113246208~75497472,192937984~25165824,239075328~4194304,247463936~96468992,348127232~62914560] 2024-09-13T00:52:14.616 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~92274688,113246208~75497472,192937984~25165824,239075328~4194304,247463936~96468992,348127232~62914560] 2024-09-13T00:52:14.633 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~96468992,113246208~104857600,239075328~171966464] 2024-09-13T00:52:14.634 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 17 diff [0~104857600,113246208~75497472,192937984~25165824,239075328~4194304,247463936~117440512,369098752~41943040] 2024-09-13T00:52:14.634 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~104857600,113246208~75497472,192937984~25165824,239075328~4194304,247463936~117440512,369098752~41943040] 2024-09-13T00:52:14.651 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~109051904,113246208~104857600,239075328~171966464] 2024-09-13T00:52:14.651 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 18 diff [0~104857600,113246208~75497472,192937984~25165824,239075328~171966464] 2024-09-13T00:52:14.651 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~104857600,113246208~75497472,192937984~25165824,239075328~171966464] 2024-09-13T00:52:14.670 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~109051904,113246208~104857600,239075328~171966464] 2024-09-13T00:52:14.670 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to 19 diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:14.670 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:14.691 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:14.692 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 2 diff [33554432~8388608,71303168~4194304,121634816~4194304,134217728~4194304,171966464~8388608,192937984~12582912,239075328~4194304,281018368~8388608,373293056~8388608] 2024-09-13T00:52:14.692 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [33554432~8388608,71303168~4194304,121634816~4194304,134217728~4194304,171966464~8388608,192937984~12582912,239075328~4194304,281018368~8388608,373293056~8388608] 2024-09-13T00:52:14.695 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [33554432~8388608,71303168~8388608,121634816~8388608,134217728~4194304,171966464~8388608,192937984~12582912,239075328~8388608,281018368~12582912,373293056~8388608] 2024-09-13T00:52:14.695 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 3 diff [0~4194304,33554432~12582912,71303168~4194304,88080384~4194304,121634816~4194304,134217728~4194304,142606336~8388608,171966464~12582912,192937984~12582912,209715200~8388608,239075328~4194304,281018368~16777216,373293056~8388608] 2024-09-13T00:52:14.695 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,33554432~12582912,71303168~4194304,88080384~4194304,121634816~4194304,134217728~4194304,142606336~8388608,171966464~12582912,192937984~12582912,209715200~8388608,239075328~4194304,281018368~16777216,373293056~8388608] 2024-09-13T00:52:14.699 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,33554432~12582912,71303168~8388608,88080384~8388608,121634816~8388608,134217728~4194304,142606336~12582912,171966464~12582912,192937984~25165824,239075328~8388608,281018368~20971520,373293056~8388608] 2024-09-13T00:52:14.699 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 4 diff [0~4194304,12582912~8388608,33554432~12582912,50331648~4194304,71303168~4194304,88080384~4194304,100663296~4194304,121634816~4194304,134217728~4194304,142606336~12582912,171966464~12582912,192937984~12582912,209715200~8388608,239075328~4194304,281018368~16777216,348127232~4194304,373293056~8388608] 2024-09-13T00:52:14.699 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~8388608,33554432~12582912,50331648~4194304,71303168~4194304,88080384~4194304,100663296~4194304,121634816~4194304,134217728~4194304,142606336~12582912,171966464~12582912,192937984~12582912,209715200~8388608,239075328~4194304,281018368~16777216,348127232~4194304,373293056~8388608] 2024-09-13T00:52:14.704 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,50331648~8388608,71303168~8388608,88080384~8388608,100663296~8388608,121634816~8388608,134217728~4194304,142606336~16777216,171966464~12582912,192937984~25165824,239075328~8388608,281018368~20971520,348127232~8388608,373293056~8388608] 2024-09-13T00:52:14.704 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 5 diff [0~4194304,12582912~8388608,33554432~12582912,50331648~8388608,71303168~8388608,88080384~4194304,96468992~8388608,117440512~8388608,134217728~20971520,171966464~12582912,192937984~12582912,209715200~8388608,239075328~4194304,260046848~12582912,281018368~16777216,348127232~4194304,373293056~8388608,398458880~8388608] 2024-09-13T00:52:14.704 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~8388608,33554432~12582912,50331648~8388608,71303168~8388608,88080384~4194304,96468992~8388608,117440512~8388608,134217728~20971520,171966464~12582912,192937984~12582912,209715200~8388608,239075328~4194304,260046848~12582912,281018368~16777216,348127232~4194304,373293056~8388608,398458880~8388608] 2024-09-13T00:52:14.711 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,50331648~12582912,71303168~12582912,88080384~20971520,117440512~12582912,134217728~25165824,171966464~12582912,192937984~25165824,239075328~8388608,260046848~12582912,281018368~20971520,348127232~8388608,373293056~8388608,398458880~8388608] 2024-09-13T00:52:14.711 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 6 diff [0~4194304,12582912~8388608,33554432~12582912,50331648~12582912,71303168~20971520,96468992~8388608,117440512~8388608,134217728~20971520,159383552~8388608,171966464~33554432,209715200~8388608,239075328~4194304,260046848~12582912,281018368~20971520,314572800~8388608,348127232~4194304,360710144~4194304,369098752~12582912,398458880~8388608] 2024-09-13T00:52:14.711 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~8388608,33554432~12582912,50331648~12582912,71303168~20971520,96468992~8388608,117440512~8388608,134217728~20971520,159383552~8388608,171966464~33554432,209715200~8388608,239075328~4194304,260046848~12582912,281018368~20971520,314572800~8388608,348127232~4194304,360710144~4194304,369098752~12582912,398458880~8388608] 2024-09-13T00:52:14.718 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,50331648~12582912,71303168~37748736,117440512~12582912,134217728~33554432,171966464~46137344,239075328~8388608,260046848~12582912,281018368~20971520,314572800~12582912,348127232~8388608,360710144~20971520,398458880~8388608] 2024-09-13T00:52:14.718 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 7 diff [0~4194304,12582912~8388608,33554432~12582912,50331648~12582912,71303168~20971520,96468992~8388608,117440512~8388608,134217728~33554432,171966464~46137344,239075328~4194304,260046848~12582912,281018368~29360128,314572800~8388608,335544320~8388608,348127232~4194304,356515840~8388608,369098752~12582912,398458880~8388608] 2024-09-13T00:52:14.718 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~8388608,33554432~12582912,50331648~12582912,71303168~20971520,96468992~8388608,117440512~8388608,134217728~33554432,171966464~46137344,239075328~4194304,260046848~12582912,281018368~29360128,314572800~8388608,335544320~8388608,348127232~4194304,356515840~8388608,369098752~12582912,398458880~8388608] 2024-09-13T00:52:14.725 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,50331648~12582912,71303168~37748736,117440512~12582912,134217728~33554432,171966464~46137344,239075328~8388608,260046848~12582912,281018368~29360128,314572800~12582912,335544320~8388608,348127232~33554432,398458880~8388608] 2024-09-13T00:52:14.725 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 8 diff [0~4194304,12582912~12582912,33554432~12582912,50331648~16777216,71303168~20971520,96468992~8388608,117440512~8388608,134217728~33554432,171966464~46137344,239075328~4194304,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~12582912,398458880~12582912] 2024-09-13T00:52:14.725 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,33554432~12582912,50331648~16777216,71303168~20971520,96468992~8388608,117440512~8388608,134217728~33554432,171966464~46137344,239075328~4194304,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~12582912,398458880~12582912] 2024-09-13T00:52:14.734 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,50331648~58720256,117440512~12582912,134217728~33554432,171966464~46137344,239075328~8388608,260046848~12582912,281018368~29360128,314572800~12582912,331350016~12582912,348127232~33554432,398458880~12582912] 2024-09-13T00:52:14.734 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 9 diff [0~4194304,12582912~12582912,33554432~12582912,50331648~16777216,71303168~20971520,96468992~8388608,117440512~8388608,134217728~33554432,171966464~46137344,239075328~4194304,247463936~4194304,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~12582912,398458880~12582912] 2024-09-13T00:52:14.735 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,33554432~12582912,50331648~16777216,71303168~20971520,96468992~8388608,117440512~8388608,134217728~33554432,171966464~46137344,239075328~4194304,247463936~4194304,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~12582912,398458880~12582912] 2024-09-13T00:52:14.744 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,50331648~58720256,117440512~12582912,134217728~33554432,171966464~46137344,239075328~16777216,260046848~16777216,281018368~29360128,314572800~12582912,331350016~12582912,348127232~33554432,398458880~12582912] 2024-09-13T00:52:14.744 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 10 diff [0~4194304,12582912~12582912,33554432~12582912,50331648~16777216,71303168~33554432,117440512~8388608,134217728~83886080,239075328~4194304,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~12582912,385875968~4194304,398458880~12582912] 2024-09-13T00:52:14.744 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,33554432~12582912,50331648~16777216,71303168~33554432,117440512~8388608,134217728~83886080,239075328~4194304,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~12582912,385875968~4194304,398458880~12582912] 2024-09-13T00:52:14.754 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,33554432~12582912,50331648~58720256,117440512~12582912,134217728~83886080,239075328~16777216,260046848~50331648,314572800~12582912,331350016~12582912,348127232~33554432,385875968~4194304,398458880~12582912] 2024-09-13T00:52:14.754 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 11 diff [0~4194304,12582912~12582912,29360128~16777216,50331648~16777216,71303168~33554432,117440512~100663296,239075328~4194304,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,327155712~16777216,348127232~4194304,356515840~8388608,369098752~20971520,394264576~16777216] 2024-09-13T00:52:14.754 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,29360128~16777216,50331648~16777216,71303168~33554432,117440512~100663296,239075328~4194304,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,327155712~16777216,348127232~4194304,356515840~8388608,369098752~20971520,394264576~16777216] 2024-09-13T00:52:14.766 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,29360128~16777216,50331648~58720256,117440512~100663296,239075328~16777216,260046848~50331648,314572800~29360128,348127232~41943040,394264576~16777216] 2024-09-13T00:52:14.766 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 12 diff [0~4194304,12582912~12582912,29360128~37748736,71303168~33554432,117440512~100663296,239075328~4194304,247463936~8388608,260046848~62914560,327155712~16777216,348127232~16777216,369098752~20971520,394264576~16777216] 2024-09-13T00:52:14.766 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,29360128~37748736,71303168~33554432,117440512~100663296,239075328~4194304,247463936~8388608,260046848~62914560,327155712~16777216,348127232~16777216,369098752~20971520,394264576~16777216] 2024-09-13T00:52:14.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:14 smithi077 bash[83127]: cluster 2024-09-13T00:52:12.852607+0000 mgr.y (mgr.25304) 490 : cluster [DBG] pgmap v1039: 105 pgs: 105 active+clean; 685 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 223 KiB/s rd, 34 MiB/s wr, 345 op/s 2024-09-13T00:52:14.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:14 smithi077 bash[83127]: cluster 2024-09-13T00:52:13.410196+0000 mon.a (mon.0) 1476 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2024-09-13T00:52:14.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:14 smithi077 bash[83127]: audit 2024-09-13T00:52:13.428468+0000 mon.a (mon.0) 1477 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:52:14.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:14 smithi077 bash[80681]: cluster 2024-09-13T00:52:12.852607+0000 mgr.y (mgr.25304) 490 : cluster [DBG] pgmap v1039: 105 pgs: 105 active+clean; 685 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 223 KiB/s rd, 34 MiB/s wr, 345 op/s 2024-09-13T00:52:14.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:14 smithi077 bash[80681]: cluster 2024-09-13T00:52:13.410196+0000 mon.a (mon.0) 1476 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2024-09-13T00:52:14.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:14 smithi077 bash[80681]: audit 2024-09-13T00:52:13.428468+0000 mon.a (mon.0) 1477 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:52:14.779 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,29360128~79691776,117440512~100663296,239075328~16777216,260046848~83886080,348127232~41943040,394264576~16777216] 2024-09-13T00:52:14.779 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 13 diff [0~4194304,12582912~54525952,71303168~33554432,117440512~100663296,239075328~4194304,247463936~8388608,260046848~83886080,348127232~16777216,369098752~20971520,394264576~16777216] 2024-09-13T00:52:14.779 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~54525952,71303168~33554432,117440512~100663296,239075328~4194304,247463936~8388608,260046848~83886080,348127232~16777216,369098752~20971520,394264576~16777216] 2024-09-13T00:52:14.792 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~96468992,117440512~100663296,239075328~16777216,260046848~83886080,348127232~41943040,394264576~16777216] 2024-09-13T00:52:14.792 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 14 diff [0~4194304,12582912~92274688,117440512~71303168,192937984~25165824,239075328~4194304,247463936~8388608,260046848~83886080,348127232~16777216,369098752~20971520,394264576~16777216] 2024-09-13T00:52:14.792 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~92274688,117440512~71303168,192937984~25165824,239075328~4194304,247463936~8388608,260046848~83886080,348127232~16777216,369098752~20971520,394264576~16777216] 2024-09-13T00:52:14.806 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~96468992,117440512~100663296,239075328~16777216,260046848~83886080,348127232~41943040,394264576~16777216] 2024-09-13T00:52:14.807 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 15 diff [0~4194304,12582912~92274688,113246208~75497472,192937984~25165824,239075328~4194304,247463936~8388608,260046848~83886080,348127232~16777216,369098752~41943040] 2024-09-13T00:52:14.807 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~92274688,113246208~75497472,192937984~25165824,239075328~4194304,247463936~8388608,260046848~83886080,348127232~16777216,369098752~41943040] 2024-09-13T00:52:14.822 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~96468992,113246208~104857600,239075328~16777216,260046848~83886080,348127232~62914560] 2024-09-13T00:52:14.823 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 16 diff [0~4194304,12582912~92274688,113246208~75497472,192937984~25165824,239075328~4194304,247463936~96468992,348127232~62914560] 2024-09-13T00:52:14.823 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~92274688,113246208~75497472,192937984~25165824,239075328~4194304,247463936~96468992,348127232~62914560] 2024-09-13T00:52:14.839 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~96468992,113246208~104857600,239075328~171966464] 2024-09-13T00:52:14.839 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 17 diff [0~104857600,113246208~75497472,192937984~25165824,239075328~4194304,247463936~117440512,369098752~41943040] 2024-09-13T00:52:14.839 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~104857600,113246208~75497472,192937984~25165824,239075328~4194304,247463936~117440512,369098752~41943040] 2024-09-13T00:52:14.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:14 smithi088 bash[68960]: cluster 2024-09-13T00:52:12.852607+0000 mgr.y (mgr.25304) 490 : cluster [DBG] pgmap v1039: 105 pgs: 105 active+clean; 685 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 223 KiB/s rd, 34 MiB/s wr, 345 op/s 2024-09-13T00:52:14.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:14 smithi088 bash[68960]: cluster 2024-09-13T00:52:13.410196+0000 mon.a (mon.0) 1476 : cluster [DBG] osdmap e930: 8 total, 8 up, 8 in 2024-09-13T00:52:14.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:14 smithi088 bash[68960]: audit 2024-09-13T00:52:13.428468+0000 mon.a (mon.0) 1477 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:52:14.857 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~109051904,113246208~104857600,239075328~171966464] 2024-09-13T00:52:14.857 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 18 diff [0~104857600,113246208~75497472,192937984~25165824,239075328~171966464] 2024-09-13T00:52:14.857 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~104857600,113246208~75497472,192937984~25165824,239075328~171966464] 2024-09-13T00:52:14.876 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~109051904,113246208~104857600,239075328~171966464] 2024-09-13T00:52:14.876 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to 19 diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:14.876 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:14.895 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:14.895 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 3 diff [0~4194304,41943040~4194304,71303168~4194304,88080384~4194304,142606336~8388608,180355072~4194304,201326592~4194304,209715200~8388608,289406976~8388608] 2024-09-13T00:52:14.896 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,41943040~4194304,71303168~4194304,88080384~4194304,142606336~8388608,180355072~4194304,201326592~4194304,209715200~8388608,289406976~8388608] 2024-09-13T00:52:14.899 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,41943040~4194304,71303168~4194304,88080384~8388608,142606336~12582912,180355072~4194304,201326592~16777216,289406976~12582912] 2024-09-13T00:52:14.899 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 4 diff [0~4194304,12582912~8388608,41943040~4194304,50331648~4194304,71303168~4194304,88080384~4194304,100663296~4194304,142606336~12582912,180355072~4194304,197132288~8388608,209715200~8388608,289406976~8388608,348127232~4194304] 2024-09-13T00:52:14.899 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~8388608,41943040~4194304,50331648~4194304,71303168~4194304,88080384~4194304,100663296~4194304,142606336~12582912,180355072~4194304,197132288~8388608,209715200~8388608,289406976~8388608,348127232~4194304] 2024-09-13T00:52:14.904 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,41943040~4194304,50331648~8388608,71303168~4194304,88080384~8388608,100663296~8388608,142606336~16777216,180355072~4194304,197132288~20971520,289406976~12582912,348127232~8388608] 2024-09-13T00:52:14.904 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 5 diff [0~4194304,12582912~8388608,41943040~4194304,50331648~8388608,71303168~8388608,88080384~4194304,96468992~8388608,117440512~4194304,138412032~16777216,176160768~8388608,197132288~8388608,209715200~8388608,260046848~12582912,289406976~8388608,348127232~4194304,398458880~8388608] 2024-09-13T00:52:14.904 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~8388608,41943040~4194304,50331648~8388608,71303168~8388608,88080384~4194304,96468992~8388608,117440512~4194304,138412032~16777216,176160768~8388608,197132288~8388608,209715200~8388608,260046848~12582912,289406976~8388608,348127232~4194304,398458880~8388608] 2024-09-13T00:52:14.909 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,41943040~4194304,50331648~12582912,71303168~12582912,88080384~20971520,117440512~4194304,138412032~20971520,176160768~8388608,197132288~20971520,260046848~12582912,289406976~12582912,348127232~8388608,398458880~8388608] 2024-09-13T00:52:14.909 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 6 diff [0~4194304,12582912~8388608,41943040~4194304,50331648~12582912,71303168~20971520,96468992~8388608,117440512~4194304,125829120~4194304,138412032~16777216,159383552~8388608,176160768~16777216,197132288~8388608,209715200~8388608,260046848~12582912,289406976~12582912,314572800~8388608,348127232~4194304,360710144~4194304,369098752~4194304,398458880~8388608] 2024-09-13T00:52:14.909 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~8388608,41943040~4194304,50331648~12582912,71303168~20971520,96468992~8388608,117440512~4194304,125829120~4194304,138412032~16777216,159383552~8388608,176160768~16777216,197132288~8388608,209715200~8388608,260046848~12582912,289406976~12582912,314572800~8388608,348127232~4194304,360710144~4194304,369098752~4194304,398458880~8388608] 2024-09-13T00:52:14.916 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,41943040~4194304,50331648~12582912,71303168~37748736,117440512~4194304,125829120~4194304,138412032~29360128,176160768~16777216,197132288~20971520,260046848~12582912,289406976~12582912,314572800~12582912,348127232~8388608,360710144~12582912,398458880~8388608] 2024-09-13T00:52:14.916 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 7 diff [0~4194304,12582912~8388608,41943040~4194304,50331648~12582912,71303168~20971520,96468992~8388608,117440512~4194304,125829120~4194304,134217728~33554432,176160768~16777216,197132288~20971520,260046848~12582912,289406976~20971520,314572800~8388608,335544320~8388608,348127232~4194304,356515840~8388608,369098752~4194304,398458880~8388608] 2024-09-13T00:52:14.916 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~8388608,41943040~4194304,50331648~12582912,71303168~20971520,96468992~8388608,117440512~4194304,125829120~4194304,134217728~33554432,176160768~16777216,197132288~20971520,260046848~12582912,289406976~20971520,314572800~8388608,335544320~8388608,348127232~4194304,356515840~8388608,369098752~4194304,398458880~8388608] 2024-09-13T00:52:14.923 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,41943040~4194304,50331648~12582912,71303168~37748736,117440512~4194304,125829120~4194304,134217728~33554432,176160768~16777216,197132288~20971520,260046848~12582912,289406976~20971520,314572800~12582912,335544320~8388608,348127232~25165824,398458880~8388608] 2024-09-13T00:52:14.923 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 8 diff [0~4194304,12582912~12582912,41943040~4194304,50331648~16777216,71303168~20971520,96468992~8388608,117440512~4194304,125829120~4194304,134217728~33554432,176160768~16777216,197132288~20971520,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~4194304,398458880~12582912] 2024-09-13T00:52:14.923 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,41943040~4194304,50331648~16777216,71303168~20971520,96468992~8388608,117440512~4194304,125829120~4194304,134217728~33554432,176160768~16777216,197132288~20971520,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~4194304,398458880~12582912] 2024-09-13T00:52:14.931 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,41943040~4194304,50331648~58720256,117440512~4194304,125829120~4194304,134217728~33554432,176160768~16777216,197132288~20971520,260046848~12582912,281018368~29360128,314572800~12582912,331350016~12582912,348127232~25165824,398458880~12582912] 2024-09-13T00:52:14.931 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 9 diff [0~4194304,12582912~12582912,37748736~8388608,50331648~16777216,71303168~20971520,96468992~8388608,117440512~4194304,125829120~4194304,134217728~33554432,176160768~41943040,247463936~4194304,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~4194304,398458880~12582912] 2024-09-13T00:52:14.931 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,37748736~8388608,50331648~16777216,71303168~20971520,96468992~8388608,117440512~4194304,125829120~4194304,134217728~33554432,176160768~41943040,247463936~4194304,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~4194304,398458880~12582912] 2024-09-13T00:52:14.940 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,37748736~8388608,50331648~58720256,117440512~4194304,125829120~4194304,134217728~33554432,176160768~41943040,247463936~8388608,260046848~16777216,281018368~29360128,314572800~12582912,331350016~12582912,348127232~25165824,398458880~12582912] 2024-09-13T00:52:14.940 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 10 diff [0~4194304,12582912~12582912,37748736~8388608,50331648~16777216,71303168~33554432,117440512~4194304,125829120~4194304,134217728~83886080,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~4194304,385875968~4194304,398458880~12582912] 2024-09-13T00:52:14.940 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,37748736~8388608,50331648~16777216,71303168~33554432,117440512~4194304,125829120~4194304,134217728~83886080,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~4194304,385875968~4194304,398458880~12582912] 2024-09-13T00:52:14.949 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,37748736~8388608,50331648~58720256,117440512~4194304,125829120~4194304,134217728~83886080,247463936~8388608,260046848~50331648,314572800~12582912,331350016~12582912,348127232~25165824,385875968~4194304,398458880~12582912] 2024-09-13T00:52:14.949 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 11 diff [0~4194304,12582912~12582912,29360128~16777216,50331648~16777216,71303168~33554432,117440512~4194304,125829120~92274688,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,327155712~16777216,348127232~4194304,356515840~8388608,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:14.949 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,29360128~16777216,50331648~16777216,71303168~33554432,117440512~4194304,125829120~92274688,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,327155712~16777216,348127232~4194304,356515840~8388608,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:14.960 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,29360128~16777216,50331648~58720256,117440512~4194304,125829120~92274688,247463936~8388608,260046848~50331648,314572800~29360128,348127232~25165824,381681664~8388608,394264576~16777216] 2024-09-13T00:52:14.961 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 12 diff [0~4194304,12582912~12582912,29360128~37748736,71303168~33554432,117440512~4194304,125829120~92274688,247463936~8388608,260046848~62914560,327155712~16777216,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:14.961 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,29360128~37748736,71303168~33554432,117440512~4194304,125829120~92274688,247463936~8388608,260046848~62914560,327155712~16777216,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:14.972 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~12582912,29360128~79691776,117440512~4194304,125829120~92274688,247463936~8388608,260046848~83886080,348127232~25165824,381681664~8388608,394264576~16777216] 2024-09-13T00:52:14.972 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 13 diff [0~4194304,12582912~54525952,71303168~33554432,117440512~4194304,125829120~92274688,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:14.973 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~54525952,71303168~33554432,117440512~4194304,125829120~92274688,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:14.986 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~96468992,117440512~4194304,125829120~92274688,247463936~8388608,260046848~83886080,348127232~29360128,381681664~8388608,394264576~16777216] 2024-09-13T00:52:14.986 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 14 diff [0~4194304,12582912~92274688,117440512~4194304,125829120~62914560,192937984~25165824,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:14.986 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~92274688,117440512~4194304,125829120~62914560,192937984~25165824,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.000 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~96468992,117440512~4194304,125829120~92274688,247463936~8388608,260046848~83886080,348127232~29360128,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.000 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 15 diff [0~4194304,12582912~92274688,113246208~8388608,125829120~62914560,192937984~25165824,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,377487360~33554432] 2024-09-13T00:52:15.000 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~92274688,113246208~8388608,125829120~62914560,192937984~25165824,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,377487360~33554432] 2024-09-13T00:52:15.015 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~96468992,113246208~8388608,125829120~92274688,247463936~8388608,260046848~83886080,348127232~62914560] 2024-09-13T00:52:15.015 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 16 diff [0~4194304,12582912~92274688,113246208~8388608,125829120~62914560,192937984~25165824,247463936~96468992,348127232~25165824,377487360~33554432] 2024-09-13T00:52:15.015 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~92274688,113246208~8388608,125829120~62914560,192937984~25165824,247463936~96468992,348127232~25165824,377487360~33554432] 2024-09-13T00:52:15.032 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~8388608,12582912~96468992,113246208~8388608,125829120~92274688,247463936~163577856] 2024-09-13T00:52:15.032 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 17 diff [0~104857600,113246208~75497472,192937984~25165824,247463936~117440512,369098752~4194304,377487360~33554432] 2024-09-13T00:52:15.032 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~104857600,113246208~75497472,192937984~25165824,247463936~117440512,369098752~4194304,377487360~33554432] 2024-09-13T00:52:15.049 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~109051904,113246208~104857600,247463936~163577856] 2024-09-13T00:52:15.049 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 18 diff [0~104857600,113246208~75497472,192937984~25165824,239075328~134217728,377487360~33554432] 2024-09-13T00:52:15.049 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~104857600,113246208~75497472,192937984~25165824,239075328~134217728,377487360~33554432] 2024-09-13T00:52:15.065 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~109051904,113246208~104857600,239075328~171966464] 2024-09-13T00:52:15.067 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to 19 diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:15.067 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:15.085 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:15.085 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 4 diff [12582912~8388608,50331648~4194304,100663296~4194304,146800640~8388608,197132288~8388608,289406976~8388608,348127232~4194304] 2024-09-13T00:52:15.086 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~8388608,50331648~4194304,100663296~4194304,146800640~8388608,197132288~8388608,289406976~8388608,348127232~4194304] 2024-09-13T00:52:15.089 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~12582912,50331648~8388608,100663296~8388608,146800640~12582912,197132288~8388608,289406976~8388608,348127232~8388608] 2024-09-13T00:52:15.089 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 5 diff [12582912~8388608,50331648~8388608,71303168~8388608,96468992~8388608,117440512~4194304,138412032~4194304,146800640~8388608,176160768~8388608,197132288~8388608,260046848~12582912,289406976~8388608,348127232~4194304,398458880~8388608] 2024-09-13T00:52:15.089 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~8388608,50331648~8388608,71303168~8388608,96468992~8388608,117440512~4194304,138412032~4194304,146800640~8388608,176160768~8388608,197132288~8388608,260046848~12582912,289406976~8388608,348127232~4194304,398458880~8388608] 2024-09-13T00:52:15.093 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~12582912,50331648~12582912,71303168~12582912,96468992~12582912,117440512~4194304,138412032~4194304,146800640~12582912,176160768~8388608,197132288~8388608,260046848~12582912,289406976~8388608,348127232~8388608,398458880~8388608] 2024-09-13T00:52:15.093 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 6 diff [12582912~8388608,50331648~12582912,71303168~16777216,96468992~8388608,117440512~4194304,125829120~4194304,138412032~4194304,146800640~8388608,159383552~8388608,176160768~16777216,197132288~8388608,260046848~12582912,289406976~12582912,314572800~8388608,348127232~4194304,360710144~4194304,369098752~4194304,398458880~8388608] 2024-09-13T00:52:15.093 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~8388608,50331648~12582912,71303168~16777216,96468992~8388608,117440512~4194304,125829120~4194304,138412032~4194304,146800640~8388608,159383552~8388608,176160768~16777216,197132288~8388608,260046848~12582912,289406976~12582912,314572800~8388608,348127232~4194304,360710144~4194304,369098752~4194304,398458880~8388608] 2024-09-13T00:52:15.099 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~12582912,50331648~12582912,71303168~16777216,96468992~12582912,117440512~4194304,125829120~4194304,138412032~4194304,146800640~20971520,176160768~16777216,197132288~8388608,260046848~12582912,289406976~12582912,314572800~12582912,348127232~8388608,360710144~12582912,398458880~8388608] 2024-09-13T00:52:15.099 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 7 diff [0~4194304,12582912~8388608,50331648~12582912,71303168~16777216,96468992~8388608,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~16777216,197132288~16777216,260046848~12582912,289406976~20971520,314572800~8388608,335544320~8388608,348127232~4194304,356515840~8388608,369098752~4194304,398458880~8388608] 2024-09-13T00:52:15.099 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~8388608,50331648~12582912,71303168~16777216,96468992~8388608,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~16777216,197132288~16777216,260046848~12582912,289406976~20971520,314572800~8388608,335544320~8388608,348127232~4194304,356515840~8388608,369098752~4194304,398458880~8388608] 2024-09-13T00:52:15.105 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~12582912,50331648~12582912,71303168~16777216,96468992~12582912,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~16777216,197132288~16777216,260046848~12582912,289406976~20971520,314572800~12582912,335544320~8388608,348127232~25165824,398458880~8388608] 2024-09-13T00:52:15.105 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 8 diff [0~4194304,12582912~12582912,50331648~16777216,71303168~16777216,96468992~8388608,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~16777216,197132288~16777216,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~4194304,398458880~12582912] 2024-09-13T00:52:15.105 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,50331648~16777216,71303168~16777216,96468992~8388608,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~16777216,197132288~16777216,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~4194304,398458880~12582912] 2024-09-13T00:52:15.112 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~12582912,50331648~37748736,96468992~12582912,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~16777216,197132288~16777216,260046848~12582912,281018368~29360128,314572800~12582912,331350016~12582912,348127232~25165824,398458880~12582912] 2024-09-13T00:52:15.112 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 9 diff [0~4194304,12582912~12582912,37748736~8388608,50331648~16777216,71303168~16777216,96468992~8388608,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~37748736,247463936~4194304,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~4194304,398458880~12582912] 2024-09-13T00:52:15.112 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,37748736~8388608,50331648~16777216,71303168~16777216,96468992~8388608,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~37748736,247463936~4194304,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~4194304,398458880~12582912] 2024-09-13T00:52:15.120 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~12582912,37748736~8388608,50331648~37748736,96468992~12582912,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~37748736,247463936~8388608,260046848~16777216,281018368~29360128,314572800~12582912,331350016~12582912,348127232~25165824,398458880~12582912] 2024-09-13T00:52:15.120 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 10 diff [0~4194304,12582912~12582912,37748736~8388608,50331648~16777216,71303168~16777216,92274688~12582912,117440512~4194304,125829120~4194304,134217728~8388608,146800640~62914560,213909504~4194304,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~4194304,385875968~4194304,398458880~12582912] 2024-09-13T00:52:15.120 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,37748736~8388608,50331648~16777216,71303168~16777216,92274688~12582912,117440512~4194304,125829120~4194304,134217728~8388608,146800640~62914560,213909504~4194304,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,331350016~12582912,348127232~4194304,356515840~8388608,369098752~4194304,385875968~4194304,398458880~12582912] 2024-09-13T00:52:15.129 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~12582912,37748736~8388608,50331648~37748736,92274688~16777216,117440512~4194304,125829120~4194304,134217728~8388608,146800640~71303168,247463936~8388608,260046848~50331648,314572800~12582912,331350016~12582912,348127232~25165824,385875968~4194304,398458880~12582912] 2024-09-13T00:52:15.129 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 11 diff [0~4194304,12582912~12582912,29360128~16777216,50331648~16777216,71303168~16777216,92274688~12582912,117440512~4194304,125829120~16777216,146800640~62914560,213909504~4194304,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,327155712~16777216,348127232~4194304,356515840~8388608,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.129 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,29360128~16777216,50331648~16777216,71303168~16777216,92274688~12582912,117440512~4194304,125829120~16777216,146800640~62914560,213909504~4194304,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,327155712~16777216,348127232~4194304,356515840~8388608,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.140 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~12582912,29360128~16777216,50331648~37748736,92274688~16777216,117440512~4194304,125829120~16777216,146800640~71303168,247463936~8388608,260046848~50331648,314572800~29360128,348127232~25165824,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.140 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 12 diff [0~4194304,12582912~12582912,29360128~37748736,71303168~16777216,92274688~12582912,117440512~4194304,125829120~16777216,146800640~62914560,213909504~4194304,247463936~8388608,260046848~62914560,327155712~16777216,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.140 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,29360128~37748736,71303168~16777216,92274688~12582912,117440512~4194304,125829120~16777216,146800640~62914560,213909504~4194304,247463936~8388608,260046848~62914560,327155712~16777216,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.152 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~12582912,29360128~79691776,117440512~4194304,125829120~16777216,146800640~71303168,247463936~8388608,260046848~83886080,348127232~25165824,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.152 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 13 diff [0~4194304,12582912~54525952,71303168~16777216,92274688~12582912,117440512~4194304,125829120~16777216,146800640~62914560,213909504~4194304,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.152 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~54525952,71303168~16777216,92274688~12582912,117440512~4194304,125829120~16777216,146800640~62914560,213909504~4194304,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.166 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~96468992,117440512~4194304,125829120~16777216,146800640~71303168,247463936~8388608,260046848~83886080,348127232~29360128,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.166 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 14 diff [0~4194304,12582912~75497472,92274688~12582912,117440512~4194304,125829120~16777216,146800640~41943040,192937984~25165824,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.166 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~75497472,92274688~12582912,117440512~4194304,125829120~16777216,146800640~41943040,192937984~25165824,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.180 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~96468992,117440512~4194304,125829120~16777216,146800640~71303168,247463936~8388608,260046848~83886080,348127232~29360128,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.180 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 15 diff [0~4194304,12582912~75497472,92274688~12582912,113246208~8388608,125829120~16777216,146800640~41943040,192937984~25165824,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,377487360~33554432] 2024-09-13T00:52:15.180 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~75497472,92274688~12582912,113246208~8388608,125829120~16777216,146800640~41943040,192937984~25165824,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,377487360~33554432] 2024-09-13T00:52:15.195 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~96468992,113246208~8388608,125829120~16777216,146800640~71303168,247463936~8388608,260046848~83886080,348127232~62914560] 2024-09-13T00:52:15.195 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 16 diff [0~4194304,12582912~75497472,92274688~12582912,113246208~8388608,125829120~16777216,146800640~41943040,192937984~25165824,247463936~96468992,348127232~25165824,377487360~33554432] 2024-09-13T00:52:15.195 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~75497472,92274688~12582912,113246208~8388608,125829120~16777216,146800640~41943040,192937984~25165824,247463936~96468992,348127232~25165824,377487360~33554432] 2024-09-13T00:52:15.211 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~96468992,113246208~8388608,125829120~16777216,146800640~71303168,247463936~163577856] 2024-09-13T00:52:15.211 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 17 diff [0~88080384,92274688~12582912,113246208~29360128,146800640~41943040,192937984~25165824,247463936~117440512,369098752~4194304,377487360~33554432] 2024-09-13T00:52:15.211 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~12582912,113246208~29360128,146800640~41943040,192937984~25165824,247463936~117440512,369098752~4194304,377487360~33554432] 2024-09-13T00:52:15.227 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~109051904,113246208~29360128,146800640~71303168,247463936~163577856] 2024-09-13T00:52:15.227 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 18 diff [0~88080384,92274688~12582912,113246208~29360128,146800640~41943040,192937984~25165824,239075328~134217728,377487360~33554432] 2024-09-13T00:52:15.227 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~12582912,113246208~29360128,146800640~41943040,192937984~25165824,239075328~134217728,377487360~33554432] 2024-09-13T00:52:15.243 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~109051904,113246208~29360128,146800640~71303168,239075328~171966464] 2024-09-13T00:52:15.244 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to 19 diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:15.244 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:15.262 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:15.263 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 5 diff [54525952~4194304,71303168~8388608,96468992~4194304,117440512~4194304,138412032~4194304,146800640~8388608,176160768~8388608,260046848~12582912,398458880~8388608] 2024-09-13T00:52:15.263 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [54525952~4194304,71303168~8388608,96468992~4194304,117440512~4194304,138412032~4194304,146800640~8388608,176160768~8388608,260046848~12582912,398458880~8388608] 2024-09-13T00:52:15.266 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [54525952~8388608,71303168~12582912,96468992~8388608,117440512~4194304,138412032~4194304,146800640~12582912,176160768~8388608,260046848~12582912,398458880~8388608] 2024-09-13T00:52:15.266 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 6 diff [50331648~12582912,71303168~16777216,96468992~4194304,117440512~4194304,125829120~4194304,138412032~4194304,146800640~8388608,159383552~8388608,176160768~16777216,260046848~12582912,297795584~4194304,314572800~8388608,360710144~4194304,369098752~4194304,398458880~8388608] 2024-09-13T00:52:15.266 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [50331648~12582912,71303168~16777216,96468992~4194304,117440512~4194304,125829120~4194304,138412032~4194304,146800640~8388608,159383552~8388608,176160768~16777216,260046848~12582912,297795584~4194304,314572800~8388608,360710144~4194304,369098752~4194304,398458880~8388608] 2024-09-13T00:52:15.271 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [50331648~12582912,71303168~16777216,96468992~8388608,117440512~4194304,125829120~4194304,138412032~4194304,146800640~20971520,176160768~16777216,260046848~12582912,297795584~4194304,314572800~12582912,360710144~12582912,398458880~8388608] 2024-09-13T00:52:15.271 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 7 diff [0~4194304,16777216~4194304,50331648~12582912,71303168~16777216,96468992~4194304,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~16777216,197132288~4194304,205520896~8388608,260046848~12582912,297795584~12582912,314572800~8388608,335544320~8388608,356515840~8388608,369098752~4194304,398458880~8388608] 2024-09-13T00:52:15.271 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,16777216~4194304,50331648~12582912,71303168~16777216,96468992~4194304,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~16777216,197132288~4194304,205520896~8388608,260046848~12582912,297795584~12582912,314572800~8388608,335544320~8388608,356515840~8388608,369098752~4194304,398458880~8388608] 2024-09-13T00:52:15.276 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,16777216~8388608,50331648~12582912,71303168~16777216,96468992~8388608,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~16777216,197132288~16777216,260046848~12582912,297795584~12582912,314572800~12582912,335544320~8388608,356515840~16777216,398458880~8388608] 2024-09-13T00:52:15.276 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 8 diff [0~4194304,16777216~8388608,50331648~16777216,71303168~16777216,96468992~4194304,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~16777216,197132288~4194304,205520896~8388608,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,356515840~8388608,369098752~4194304,398458880~12582912] 2024-09-13T00:52:15.276 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,16777216~8388608,50331648~16777216,71303168~16777216,96468992~4194304,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~16777216,197132288~4194304,205520896~8388608,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,356515840~8388608,369098752~4194304,398458880~12582912] 2024-09-13T00:52:15.282 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,16777216~8388608,50331648~37748736,96468992~8388608,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~16777216,197132288~16777216,260046848~12582912,281018368~29360128,314572800~12582912,331350016~12582912,356515840~16777216,398458880~12582912] 2024-09-13T00:52:15.282 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 9 diff [0~4194304,16777216~8388608,37748736~8388608,50331648~16777216,71303168~16777216,96468992~4194304,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~37748736,247463936~4194304,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,356515840~8388608,369098752~4194304,398458880~12582912] 2024-09-13T00:52:15.282 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,16777216~8388608,37748736~8388608,50331648~16777216,71303168~16777216,96468992~4194304,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~37748736,247463936~4194304,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,356515840~8388608,369098752~4194304,398458880~12582912] 2024-09-13T00:52:15.289 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,16777216~8388608,37748736~8388608,50331648~37748736,96468992~8388608,117440512~4194304,125829120~4194304,134217728~8388608,146800640~20971520,176160768~37748736,247463936~8388608,260046848~16777216,281018368~29360128,314572800~12582912,331350016~12582912,356515840~16777216,398458880~12582912] 2024-09-13T00:52:15.289 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 10 diff [0~4194304,16777216~8388608,37748736~8388608,50331648~16777216,71303168~16777216,92274688~8388608,117440512~4194304,125829120~4194304,134217728~8388608,146800640~62914560,213909504~4194304,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,331350016~12582912,356515840~8388608,369098752~4194304,385875968~4194304,398458880~12582912] 2024-09-13T00:52:15.289 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,16777216~8388608,37748736~8388608,50331648~16777216,71303168~16777216,92274688~8388608,117440512~4194304,125829120~4194304,134217728~8388608,146800640~62914560,213909504~4194304,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,331350016~12582912,356515840~8388608,369098752~4194304,385875968~4194304,398458880~12582912] 2024-09-13T00:52:15.297 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,16777216~8388608,37748736~8388608,50331648~37748736,92274688~12582912,117440512~4194304,125829120~4194304,134217728~8388608,146800640~71303168,247463936~8388608,260046848~50331648,314572800~12582912,331350016~12582912,356515840~16777216,385875968~4194304,398458880~12582912] 2024-09-13T00:52:15.297 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 11 diff [0~4194304,12582912~12582912,29360128~16777216,50331648~16777216,71303168~16777216,92274688~8388608,117440512~4194304,125829120~16777216,146800640~62914560,213909504~4194304,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,327155712~16777216,348127232~4194304,356515840~8388608,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.297 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,29360128~16777216,50331648~16777216,71303168~16777216,92274688~8388608,117440512~4194304,125829120~16777216,146800640~62914560,213909504~4194304,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,327155712~16777216,348127232~4194304,356515840~8388608,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.306 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~12582912,29360128~16777216,50331648~37748736,92274688~12582912,117440512~4194304,125829120~16777216,146800640~71303168,247463936~8388608,260046848~50331648,314572800~29360128,348127232~25165824,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.306 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 12 diff [0~4194304,12582912~12582912,29360128~37748736,71303168~16777216,92274688~8388608,117440512~4194304,125829120~16777216,146800640~62914560,213909504~4194304,247463936~8388608,260046848~62914560,327155712~16777216,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.306 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,29360128~37748736,71303168~16777216,92274688~8388608,117440512~4194304,125829120~16777216,146800640~62914560,213909504~4194304,247463936~8388608,260046848~62914560,327155712~16777216,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.316 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~12582912,29360128~75497472,117440512~4194304,125829120~16777216,146800640~71303168,247463936~8388608,260046848~83886080,348127232~25165824,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.316 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 13 diff [0~4194304,12582912~54525952,71303168~16777216,92274688~8388608,104857600~4194304,117440512~4194304,125829120~16777216,146800640~62914560,213909504~4194304,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.316 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~54525952,71303168~16777216,92274688~8388608,104857600~4194304,117440512~4194304,125829120~16777216,146800640~62914560,213909504~4194304,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.328 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~96468992,117440512~4194304,125829120~16777216,146800640~71303168,247463936~8388608,260046848~83886080,348127232~29360128,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.328 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 14 diff [0~4194304,12582912~37748736,54525952~33554432,92274688~8388608,104857600~4194304,117440512~4194304,125829120~16777216,146800640~71303168,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.328 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~37748736,54525952~33554432,92274688~8388608,104857600~4194304,117440512~4194304,125829120~16777216,146800640~71303168,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.339 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~96468992,117440512~4194304,125829120~16777216,146800640~71303168,247463936~8388608,260046848~83886080,348127232~29360128,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.339 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 15 diff [0~4194304,12582912~37748736,54525952~33554432,92274688~8388608,104857600~4194304,113246208~8388608,125829120~16777216,146800640~71303168,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,377487360~33554432] 2024-09-13T00:52:15.339 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~37748736,54525952~33554432,92274688~8388608,104857600~4194304,113246208~8388608,125829120~16777216,146800640~71303168,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,377487360~33554432] 2024-09-13T00:52:15.352 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~96468992,113246208~8388608,125829120~16777216,146800640~71303168,247463936~8388608,260046848~83886080,348127232~62914560] 2024-09-13T00:52:15.352 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 16 diff [0~4194304,12582912~37748736,54525952~33554432,92274688~8388608,104857600~4194304,113246208~8388608,125829120~16777216,146800640~71303168,247463936~96468992,348127232~25165824,377487360~33554432] 2024-09-13T00:52:15.352 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~37748736,54525952~33554432,92274688~8388608,104857600~4194304,113246208~8388608,125829120~16777216,146800640~71303168,247463936~96468992,348127232~25165824,377487360~33554432] 2024-09-13T00:52:15.366 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~96468992,113246208~8388608,125829120~16777216,146800640~71303168,247463936~163577856] 2024-09-13T00:52:15.366 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 17 diff [0~50331648,54525952~33554432,92274688~8388608,104857600~4194304,113246208~29360128,146800640~71303168,247463936~117440512,369098752~4194304,377487360~33554432] 2024-09-13T00:52:15.366 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~50331648,54525952~33554432,92274688~8388608,104857600~4194304,113246208~29360128,146800640~71303168,247463936~117440512,369098752~4194304,377487360~33554432] 2024-09-13T00:52:15.380 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~109051904,113246208~29360128,146800640~71303168,247463936~163577856] 2024-09-13T00:52:15.380 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 18 diff [0~88080384,92274688~8388608,104857600~4194304,113246208~29360128,146800640~71303168,239075328~134217728,377487360~33554432] 2024-09-13T00:52:15.380 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~4194304,113246208~29360128,146800640~71303168,239075328~134217728,377487360~33554432] 2024-09-13T00:52:15.394 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~109051904,113246208~29360128,146800640~71303168,239075328~171966464] 2024-09-13T00:52:15.396 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to 19 diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:15.396 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:15.412 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:15.412 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 6 diff [50331648~8388608,79691776~8388608,125829120~4194304,159383552~8388608,184549376~8388608,297795584~4194304,314572800~8388608,360710144~4194304,369098752~4194304] 2024-09-13T00:52:15.412 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [50331648~8388608,79691776~8388608,125829120~4194304,159383552~8388608,184549376~8388608,297795584~4194304,314572800~8388608,360710144~4194304,369098752~4194304] 2024-09-13T00:52:15.415 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [50331648~12582912,79691776~8388608,125829120~4194304,159383552~8388608,184549376~8388608,297795584~4194304,314572800~12582912,360710144~12582912] 2024-09-13T00:52:15.415 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 7 diff [0~4194304,16777216~4194304,50331648~8388608,79691776~8388608,125829120~4194304,134217728~4194304,155189248~12582912,184549376~8388608,197132288~4194304,205520896~8388608,297795584~12582912,314572800~8388608,335544320~8388608,356515840~8388608,369098752~4194304] 2024-09-13T00:52:15.415 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,16777216~4194304,50331648~8388608,79691776~8388608,125829120~4194304,134217728~4194304,155189248~12582912,184549376~8388608,197132288~4194304,205520896~8388608,297795584~12582912,314572800~8388608,335544320~8388608,356515840~8388608,369098752~4194304] 2024-09-13T00:52:15.420 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,16777216~8388608,50331648~12582912,79691776~8388608,125829120~4194304,134217728~4194304,155189248~12582912,184549376~8388608,197132288~16777216,297795584~12582912,314572800~12582912,335544320~8388608,356515840~16777216] 2024-09-13T00:52:15.420 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 8 diff [0~4194304,16777216~8388608,50331648~8388608,62914560~4194304,79691776~8388608,96468992~4194304,125829120~4194304,134217728~4194304,155189248~12582912,184549376~8388608,197132288~4194304,205520896~8388608,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,356515840~8388608,369098752~4194304,398458880~8388608] 2024-09-13T00:52:15.420 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,16777216~8388608,50331648~8388608,62914560~4194304,79691776~8388608,96468992~4194304,125829120~4194304,134217728~4194304,155189248~12582912,184549376~8388608,197132288~4194304,205520896~8388608,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,356515840~8388608,369098752~4194304,398458880~8388608] 2024-09-13T00:52:15.425 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,16777216~8388608,50331648~20971520,79691776~8388608,96468992~8388608,125829120~4194304,134217728~4194304,155189248~12582912,184549376~8388608,197132288~16777216,260046848~12582912,281018368~29360128,314572800~12582912,331350016~12582912,356515840~16777216,398458880~12582912] 2024-09-13T00:52:15.425 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 9 diff [0~4194304,16777216~8388608,37748736~8388608,50331648~8388608,62914560~4194304,79691776~8388608,96468992~4194304,125829120~4194304,134217728~4194304,155189248~12582912,184549376~29360128,247463936~4194304,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,356515840~8388608,369098752~4194304,398458880~8388608] 2024-09-13T00:52:15.425 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,16777216~8388608,37748736~8388608,50331648~8388608,62914560~4194304,79691776~8388608,96468992~4194304,125829120~4194304,134217728~4194304,155189248~12582912,184549376~29360128,247463936~4194304,260046848~12582912,281018368~29360128,314572800~8388608,331350016~12582912,356515840~8388608,369098752~4194304,398458880~8388608] 2024-09-13T00:52:15.432 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,16777216~8388608,37748736~8388608,50331648~20971520,79691776~8388608,96468992~8388608,125829120~4194304,134217728~4194304,155189248~12582912,184549376~29360128,247463936~8388608,260046848~16777216,281018368~29360128,314572800~12582912,331350016~12582912,356515840~16777216,398458880~12582912] 2024-09-13T00:52:15.432 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 10 diff [0~4194304,16777216~8388608,37748736~8388608,50331648~8388608,62914560~4194304,79691776~8388608,92274688~8388608,125829120~4194304,134217728~4194304,155189248~25165824,184549376~25165824,213909504~4194304,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,331350016~12582912,356515840~8388608,369098752~4194304,385875968~4194304,398458880~8388608] 2024-09-13T00:52:15.432 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,16777216~8388608,37748736~8388608,50331648~8388608,62914560~4194304,79691776~8388608,92274688~8388608,125829120~4194304,134217728~4194304,155189248~25165824,184549376~25165824,213909504~4194304,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,331350016~12582912,356515840~8388608,369098752~4194304,385875968~4194304,398458880~8388608] 2024-09-13T00:52:15.438 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,16777216~8388608,37748736~8388608,50331648~20971520,79691776~8388608,92274688~12582912,125829120~4194304,134217728~4194304,155189248~62914560,247463936~8388608,260046848~50331648,314572800~12582912,331350016~12582912,356515840~16777216,385875968~4194304,398458880~12582912] 2024-09-13T00:52:15.438 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 11 diff [0~4194304,12582912~12582912,29360128~16777216,50331648~8388608,62914560~4194304,79691776~8388608,92274688~8388608,125829120~12582912,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,327155712~16777216,348127232~4194304,356515840~8388608,369098752~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.438 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,29360128~16777216,50331648~8388608,62914560~4194304,79691776~8388608,92274688~8388608,125829120~12582912,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,260046848~16777216,281018368~29360128,314572800~8388608,327155712~16777216,348127232~4194304,356515840~8388608,369098752~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.445 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~12582912,29360128~16777216,50331648~20971520,79691776~8388608,92274688~12582912,125829120~12582912,155189248~62914560,247463936~8388608,260046848~50331648,314572800~29360128,348127232~25165824,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.446 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 12 diff [0~4194304,12582912~12582912,29360128~29360128,62914560~4194304,71303168~16777216,92274688~8388608,125829120~12582912,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,260046848~62914560,327155712~16777216,348127232~16777216,369098752~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.446 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,29360128~29360128,62914560~4194304,71303168~16777216,92274688~8388608,125829120~12582912,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,260046848~62914560,327155712~16777216,348127232~16777216,369098752~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.454 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~12582912,29360128~75497472,125829120~12582912,155189248~62914560,247463936~8388608,260046848~83886080,348127232~25165824,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.454 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 13 diff [0~4194304,12582912~54525952,71303168~16777216,92274688~8388608,104857600~4194304,125829120~16777216,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.454 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~54525952,71303168~16777216,92274688~8388608,104857600~4194304,125829120~16777216,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.465 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~96468992,125829120~16777216,155189248~62914560,247463936~8388608,260046848~83886080,348127232~29360128,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.465 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 14 diff [0~4194304,12582912~37748736,54525952~33554432,92274688~8388608,104857600~4194304,125829120~16777216,155189248~62914560,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.465 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~37748736,54525952~33554432,92274688~8388608,104857600~4194304,125829120~16777216,155189248~62914560,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.476 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~96468992,125829120~16777216,155189248~62914560,247463936~8388608,260046848~83886080,348127232~29360128,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.476 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 15 diff [0~4194304,12582912~37748736,54525952~33554432,92274688~8388608,104857600~4194304,113246208~4194304,125829120~16777216,155189248~62914560,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,377487360~29360128] 2024-09-13T00:52:15.476 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~37748736,54525952~33554432,92274688~8388608,104857600~4194304,113246208~4194304,125829120~16777216,155189248~62914560,247463936~8388608,260046848~83886080,348127232~16777216,369098752~4194304,377487360~29360128] 2024-09-13T00:52:15.488 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~96468992,113246208~8388608,125829120~16777216,155189248~62914560,247463936~8388608,260046848~83886080,348127232~62914560] 2024-09-13T00:52:15.488 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 16 diff [0~4194304,12582912~37748736,54525952~33554432,92274688~8388608,104857600~4194304,113246208~8388608,125829120~16777216,155189248~62914560,247463936~96468992,348127232~25165824,377487360~29360128] 2024-09-13T00:52:15.488 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~37748736,54525952~33554432,92274688~8388608,104857600~4194304,113246208~8388608,125829120~16777216,155189248~62914560,247463936~96468992,348127232~25165824,377487360~29360128] 2024-09-13T00:52:15.502 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~96468992,113246208~8388608,125829120~16777216,155189248~62914560,247463936~163577856] 2024-09-13T00:52:15.502 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 17 diff [0~50331648,54525952~33554432,92274688~8388608,104857600~4194304,113246208~29360128,155189248~62914560,247463936~117440512,369098752~4194304,377487360~29360128] 2024-09-13T00:52:15.502 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~50331648,54525952~33554432,92274688~8388608,104857600~4194304,113246208~29360128,155189248~62914560,247463936~117440512,369098752~4194304,377487360~29360128] 2024-09-13T00:52:15.516 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~109051904,113246208~29360128,155189248~62914560,247463936~163577856] 2024-09-13T00:52:15.516 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 18 diff [0~88080384,92274688~8388608,104857600~4194304,113246208~29360128,155189248~62914560,239075328~134217728,377487360~29360128] 2024-09-13T00:52:15.516 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~4194304,113246208~29360128,155189248~62914560,239075328~134217728,377487360~29360128] 2024-09-13T00:52:15.531 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~109051904,113246208~29360128,155189248~62914560,239075328~171966464] 2024-09-13T00:52:15.531 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to 19 diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:15.531 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:15.548 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:15.548 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 7 diff [0~4194304,16777216~4194304,83886080~4194304,134217728~4194304,155189248~4194304,197132288~4194304,205520896~8388608,301989888~8388608,335544320~8388608,356515840~4194304] 2024-09-13T00:52:15.548 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,16777216~4194304,83886080~4194304,134217728~4194304,155189248~4194304,197132288~4194304,205520896~8388608,301989888~8388608,335544320~8388608,356515840~4194304] 2024-09-13T00:52:15.551 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,16777216~8388608,83886080~4194304,134217728~4194304,155189248~8388608,197132288~16777216,301989888~8388608,335544320~8388608,356515840~4194304] 2024-09-13T00:52:15.552 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 8 diff [0~4194304,16777216~8388608,62914560~4194304,79691776~8388608,96468992~4194304,134217728~4194304,155189248~4194304,197132288~4194304,205520896~8388608,260046848~12582912,281018368~16777216,301989888~8388608,331350016~12582912,356515840~4194304,398458880~8388608] 2024-09-13T00:52:15.552 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,16777216~8388608,62914560~4194304,79691776~8388608,96468992~4194304,134217728~4194304,155189248~4194304,197132288~4194304,205520896~8388608,260046848~12582912,281018368~16777216,301989888~8388608,331350016~12582912,356515840~4194304,398458880~8388608] 2024-09-13T00:52:15.556 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,16777216~8388608,62914560~8388608,79691776~8388608,96468992~8388608,134217728~4194304,155189248~8388608,197132288~16777216,260046848~12582912,281018368~29360128,331350016~12582912,356515840~4194304,398458880~12582912] 2024-09-13T00:52:15.556 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 9 diff [0~4194304,16777216~8388608,37748736~8388608,62914560~4194304,79691776~8388608,96468992~4194304,134217728~4194304,155189248~4194304,192937984~20971520,247463936~4194304,260046848~12582912,281018368~16777216,301989888~8388608,331350016~12582912,356515840~4194304,398458880~8388608] 2024-09-13T00:52:15.556 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,16777216~8388608,37748736~8388608,62914560~4194304,79691776~8388608,96468992~4194304,134217728~4194304,155189248~4194304,192937984~20971520,247463936~4194304,260046848~12582912,281018368~16777216,301989888~8388608,331350016~12582912,356515840~4194304,398458880~8388608] 2024-09-13T00:52:15.561 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,16777216~8388608,37748736~8388608,62914560~8388608,79691776~8388608,96468992~8388608,134217728~4194304,155189248~8388608,192937984~20971520,247463936~8388608,260046848~16777216,281018368~29360128,331350016~12582912,356515840~4194304,398458880~12582912] 2024-09-13T00:52:15.561 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 10 diff [0~4194304,16777216~8388608,37748736~8388608,62914560~4194304,79691776~8388608,92274688~8388608,134217728~4194304,155189248~8388608,167772160~12582912,192937984~16777216,213909504~4194304,247463936~8388608,260046848~16777216,281018368~16777216,301989888~8388608,314572800~4194304,331350016~12582912,356515840~4194304,385875968~4194304,398458880~8388608] 2024-09-13T00:52:15.561 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,16777216~8388608,37748736~8388608,62914560~4194304,79691776~8388608,92274688~8388608,134217728~4194304,155189248~8388608,167772160~12582912,192937984~16777216,213909504~4194304,247463936~8388608,260046848~16777216,281018368~16777216,301989888~8388608,314572800~4194304,331350016~12582912,356515840~4194304,385875968~4194304,398458880~8388608] 2024-09-13T00:52:15.566 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,16777216~8388608,37748736~8388608,62914560~8388608,79691776~8388608,92274688~12582912,134217728~4194304,155189248~29360128,192937984~25165824,247463936~8388608,260046848~50331648,314572800~8388608,331350016~12582912,356515840~4194304,385875968~4194304,398458880~12582912] 2024-09-13T00:52:15.566 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 11 diff [0~4194304,12582912~12582912,29360128~16777216,62914560~4194304,79691776~8388608,92274688~8388608,125829120~12582912,155189248~8388608,167772160~16777216,192937984~16777216,213909504~4194304,247463936~8388608,260046848~16777216,281018368~16777216,301989888~8388608,314572800~4194304,327155712~16777216,348127232~4194304,356515840~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.566 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,29360128~16777216,62914560~4194304,79691776~8388608,92274688~8388608,125829120~12582912,155189248~8388608,167772160~16777216,192937984~16777216,213909504~4194304,247463936~8388608,260046848~16777216,281018368~16777216,301989888~8388608,314572800~4194304,327155712~16777216,348127232~4194304,356515840~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.573 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~12582912,29360128~16777216,62914560~8388608,79691776~8388608,92274688~12582912,125829120~12582912,155189248~33554432,192937984~25165824,247463936~8388608,260046848~50331648,314572800~8388608,327155712~16777216,348127232~12582912,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.573 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 12 diff [0~4194304,12582912~12582912,29360128~25165824,62914560~4194304,71303168~29360128,125829120~12582912,155189248~8388608,167772160~20971520,192937984~16777216,213909504~4194304,247463936~8388608,260046848~37748736,301989888~16777216,327155712~16777216,348127232~12582912,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.573 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~12582912,29360128~25165824,62914560~4194304,71303168~29360128,125829120~12582912,155189248~8388608,167772160~20971520,192937984~16777216,213909504~4194304,247463936~8388608,260046848~37748736,301989888~16777216,327155712~16777216,348127232~12582912,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.581 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~12582912,29360128~25165824,62914560~41943040,125829120~12582912,155189248~62914560,247463936~8388608,260046848~62914560,327155712~16777216,348127232~12582912,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.581 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 13 diff [0~4194304,12582912~41943040,58720256~8388608,71303168~29360128,104857600~4194304,125829120~16777216,155189248~8388608,167772160~20971520,192937984~16777216,213909504~4194304,247463936~8388608,260046848~37748736,301989888~16777216,322961408~20971520,348127232~12582912,369098752~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.581 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~41943040,58720256~8388608,71303168~29360128,104857600~4194304,125829120~16777216,155189248~8388608,167772160~20971520,192937984~16777216,213909504~4194304,247463936~8388608,260046848~37748736,301989888~16777216,322961408~20971520,348127232~12582912,369098752~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.590 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~41943040,58720256~50331648,125829120~16777216,155189248~62914560,247463936~8388608,260046848~83886080,348127232~12582912,369098752~8388608,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.590 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 14 diff [0~4194304,12582912~37748736,54525952~12582912,71303168~29360128,104857600~4194304,125829120~16777216,155189248~33554432,192937984~25165824,247463936~8388608,260046848~37748736,301989888~16777216,322961408~20971520,348127232~12582912,369098752~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.590 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~37748736,54525952~12582912,71303168~29360128,104857600~4194304,125829120~16777216,155189248~33554432,192937984~25165824,247463936~8388608,260046848~37748736,301989888~16777216,322961408~20971520,348127232~12582912,369098752~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.601 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~96468992,125829120~16777216,155189248~62914560,247463936~8388608,260046848~83886080,348127232~12582912,369098752~8388608,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.604 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 15 diff [0~4194304,12582912~37748736,54525952~12582912,71303168~29360128,104857600~4194304,113246208~4194304,125829120~16777216,155189248~33554432,192937984~25165824,247463936~8388608,260046848~37748736,301989888~16777216,322961408~20971520,348127232~12582912,369098752~4194304,377487360~29360128] 2024-09-13T00:52:15.604 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~37748736,54525952~12582912,71303168~29360128,104857600~4194304,113246208~4194304,125829120~16777216,155189248~33554432,192937984~25165824,247463936~8388608,260046848~37748736,301989888~16777216,322961408~20971520,348127232~12582912,369098752~4194304,377487360~29360128] 2024-09-13T00:52:15.612 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~96468992,113246208~8388608,125829120~16777216,155189248~62914560,247463936~8388608,260046848~83886080,348127232~12582912,369098752~41943040] 2024-09-13T00:52:15.612 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 16 diff [0~4194304,12582912~37748736,54525952~12582912,71303168~29360128,104857600~4194304,113246208~8388608,125829120~16777216,155189248~33554432,192937984~25165824,247463936~50331648,301989888~41943040,348127232~16777216,369098752~4194304,377487360~29360128] 2024-09-13T00:52:15.612 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~4194304,12582912~37748736,54525952~12582912,71303168~29360128,104857600~4194304,113246208~8388608,125829120~16777216,155189248~33554432,192937984~25165824,247463936~50331648,301989888~41943040,348127232~16777216,369098752~4194304,377487360~29360128] 2024-09-13T00:52:15.625 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~4194304,12582912~96468992,113246208~8388608,125829120~16777216,155189248~62914560,247463936~163577856] 2024-09-13T00:52:15.625 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 17 diff [0~50331648,54525952~12582912,71303168~29360128,104857600~4194304,113246208~29360128,155189248~33554432,192937984~25165824,247463936~50331648,301989888~62914560,369098752~4194304,377487360~29360128] 2024-09-13T00:52:15.625 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~50331648,54525952~12582912,71303168~29360128,104857600~4194304,113246208~29360128,155189248~33554432,192937984~25165824,247463936~50331648,301989888~62914560,369098752~4194304,377487360~29360128] 2024-09-13T00:52:15.637 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~109051904,113246208~29360128,155189248~62914560,247463936~163577856] 2024-09-13T00:52:15.638 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 18 diff [0~67108864,71303168~29360128,104857600~4194304,113246208~29360128,155189248~33554432,192937984~25165824,239075328~58720256,301989888~71303168,377487360~29360128] 2024-09-13T00:52:15.638 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~67108864,71303168~29360128,104857600~4194304,113246208~29360128,155189248~33554432,192937984~25165824,239075328~58720256,301989888~71303168,377487360~29360128] 2024-09-13T00:52:15.651 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~109051904,113246208~29360128,155189248~62914560,239075328~171966464] 2024-09-13T00:52:15.651 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to 19 diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:15.651 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:15.666 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:15.666 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 8 diff [20971520~4194304,62914560~4194304,79691776~4194304,96468992~4194304,260046848~12582912,281018368~16777216,331350016~8388608,398458880~8388608] 2024-09-13T00:52:15.667 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [20971520~4194304,62914560~4194304,79691776~4194304,96468992~4194304,260046848~12582912,281018368~16777216,331350016~8388608,398458880~8388608] 2024-09-13T00:52:15.669 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [20971520~4194304,62914560~8388608,79691776~4194304,96468992~8388608,260046848~12582912,281018368~20971520,331350016~8388608,398458880~12582912] 2024-09-13T00:52:15.669 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 9 diff [20971520~4194304,37748736~8388608,62914560~4194304,79691776~4194304,96468992~4194304,192937984~12582912,247463936~4194304,260046848~12582912,281018368~16777216,301989888~4194304,331350016~8388608,398458880~8388608] 2024-09-13T00:52:15.669 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [20971520~4194304,37748736~8388608,62914560~4194304,79691776~4194304,96468992~4194304,192937984~12582912,247463936~4194304,260046848~12582912,281018368~16777216,301989888~4194304,331350016~8388608,398458880~8388608] 2024-09-13T00:52:15.673 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [20971520~4194304,37748736~8388608,62914560~8388608,79691776~4194304,96468992~8388608,192937984~12582912,247463936~8388608,260046848~16777216,281018368~29360128,331350016~8388608,398458880~12582912] 2024-09-13T00:52:15.673 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 10 diff [20971520~4194304,37748736~8388608,62914560~4194304,79691776~4194304,92274688~8388608,155189248~8388608,167772160~12582912,192937984~16777216,213909504~4194304,247463936~8388608,260046848~16777216,281018368~16777216,301989888~4194304,314572800~4194304,331350016~8388608,385875968~4194304,398458880~8388608] 2024-09-13T00:52:15.673 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [20971520~4194304,37748736~8388608,62914560~4194304,79691776~4194304,92274688~8388608,155189248~8388608,167772160~12582912,192937984~16777216,213909504~4194304,247463936~8388608,260046848~16777216,281018368~16777216,301989888~4194304,314572800~4194304,331350016~8388608,385875968~4194304,398458880~8388608] 2024-09-13T00:52:15.678 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [20971520~4194304,37748736~8388608,62914560~8388608,79691776~4194304,92274688~12582912,155189248~29360128,192937984~25165824,247463936~8388608,260046848~50331648,314572800~8388608,331350016~8388608,385875968~4194304,398458880~12582912] 2024-09-13T00:52:15.678 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 11 diff [12582912~4194304,20971520~4194304,29360128~16777216,62914560~4194304,79691776~4194304,92274688~8388608,125829120~8388608,155189248~8388608,167772160~16777216,192937984~16777216,213909504~4194304,247463936~8388608,260046848~16777216,281018368~16777216,301989888~4194304,314572800~4194304,327155712~12582912,348127232~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.678 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~4194304,20971520~4194304,29360128~16777216,62914560~4194304,79691776~4194304,92274688~8388608,125829120~8388608,155189248~8388608,167772160~16777216,192937984~16777216,213909504~4194304,247463936~8388608,260046848~16777216,281018368~16777216,301989888~4194304,314572800~4194304,327155712~12582912,348127232~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.685 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~12582912,29360128~16777216,62914560~8388608,79691776~4194304,92274688~12582912,125829120~8388608,155189248~33554432,192937984~25165824,247463936~8388608,260046848~50331648,314572800~8388608,327155712~12582912,348127232~8388608,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.685 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 12 diff [12582912~4194304,20971520~4194304,29360128~25165824,62914560~4194304,71303168~16777216,92274688~8388608,125829120~8388608,155189248~8388608,167772160~20971520,192937984~16777216,213909504~4194304,247463936~8388608,260046848~37748736,301989888~16777216,327155712~12582912,348127232~8388608,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.685 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~4194304,20971520~4194304,29360128~25165824,62914560~4194304,71303168~16777216,92274688~8388608,125829120~8388608,155189248~8388608,167772160~20971520,192937984~16777216,213909504~4194304,247463936~8388608,260046848~37748736,301989888~16777216,327155712~12582912,348127232~8388608,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.691 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~12582912,29360128~25165824,62914560~41943040,125829120~8388608,155189248~62914560,247463936~8388608,260046848~62914560,327155712~12582912,348127232~8388608,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.691 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 13 diff [12582912~41943040,58720256~8388608,71303168~16777216,92274688~8388608,104857600~4194304,125829120~8388608,138412032~4194304,155189248~8388608,167772160~20971520,192937984~16777216,213909504~4194304,247463936~8388608,260046848~37748736,301989888~16777216,322961408~16777216,348127232~8388608,369098752~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.691 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~41943040,58720256~8388608,71303168~16777216,92274688~8388608,104857600~4194304,125829120~8388608,138412032~4194304,155189248~8388608,167772160~20971520,192937984~16777216,213909504~4194304,247463936~8388608,260046848~37748736,301989888~16777216,322961408~16777216,348127232~8388608,369098752~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.699 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~41943040,58720256~50331648,125829120~8388608,138412032~4194304,155189248~62914560,247463936~8388608,260046848~79691776,348127232~8388608,369098752~8388608,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.699 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 14 diff [12582912~37748736,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,260046848~37748736,301989888~16777216,322961408~16777216,348127232~8388608,369098752~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.699 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~37748736,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,260046848~37748736,301989888~16777216,322961408~16777216,348127232~8388608,369098752~4194304,381681664~8388608,394264576~12582912] 2024-09-13T00:52:15.708 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~96468992,125829120~8388608,138412032~4194304,155189248~62914560,247463936~8388608,260046848~79691776,348127232~8388608,369098752~8388608,381681664~8388608,394264576~16777216] 2024-09-13T00:52:15.708 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 15 diff [12582912~37748736,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~4194304,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,260046848~37748736,301989888~16777216,322961408~16777216,348127232~8388608,369098752~4194304,377487360~29360128] 2024-09-13T00:52:15.708 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~37748736,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~4194304,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,260046848~37748736,301989888~16777216,322961408~16777216,348127232~8388608,369098752~4194304,377487360~29360128] 2024-09-13T00:52:15.718 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~96468992,113246208~8388608,125829120~8388608,138412032~4194304,155189248~62914560,247463936~8388608,260046848~79691776,348127232~8388608,369098752~41943040] 2024-09-13T00:52:15.718 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 16 diff [12582912~37748736,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~8388608,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~50331648,301989888~41943040,348127232~8388608,360710144~4194304,369098752~4194304,377487360~29360128] 2024-09-13T00:52:15.718 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~37748736,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~8388608,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~50331648,301989888~41943040,348127232~8388608,360710144~4194304,369098752~4194304,377487360~29360128] 2024-09-13T00:52:15.729 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~96468992,113246208~8388608,125829120~8388608,138412032~4194304,155189248~62914560,247463936~109051904,360710144~50331648] 2024-09-13T00:52:15.729 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 17 diff [4194304~46137344,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~20971520,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~50331648,301989888~62914560,369098752~4194304,377487360~29360128] 2024-09-13T00:52:15.729 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~46137344,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~20971520,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~50331648,301989888~62914560,369098752~4194304,377487360~29360128] 2024-09-13T00:52:15.741 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~104857600,113246208~20971520,138412032~4194304,155189248~62914560,247463936~163577856] 2024-09-13T00:52:15.741 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 18 diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~4194304,113246208~20971520,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,239075328~58720256,301989888~71303168,377487360~29360128] 2024-09-13T00:52:15.741 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~4194304,113246208~20971520,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,239075328~58720256,301989888~71303168,377487360~29360128] 2024-09-13T00:52:15.752 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~104857600,113246208~20971520,138412032~4194304,155189248~62914560,239075328~171966464] 2024-09-13T00:52:15.753 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to 19 diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:15.753 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:15.767 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:15.767 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 9 diff [37748736~8388608,79691776~4194304,192937984~12582912,247463936~4194304,260046848~12582912,285212672~8388608,301989888~4194304] 2024-09-13T00:52:15.767 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [37748736~8388608,79691776~4194304,192937984~12582912,247463936~4194304,260046848~12582912,285212672~8388608,301989888~4194304] 2024-09-13T00:52:15.770 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [37748736~8388608,79691776~4194304,192937984~12582912,247463936~8388608,260046848~16777216,285212672~8388608,301989888~8388608] 2024-09-13T00:52:15.770 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 10 diff [37748736~8388608,79691776~4194304,92274688~8388608,155189248~8388608,167772160~12582912,192937984~16777216,213909504~4194304,247463936~8388608,260046848~16777216,285212672~8388608,301989888~4194304,314572800~4194304,385875968~4194304] 2024-09-13T00:52:15.770 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [37748736~8388608,79691776~4194304,92274688~8388608,155189248~8388608,167772160~12582912,192937984~16777216,213909504~4194304,247463936~8388608,260046848~16777216,285212672~8388608,301989888~4194304,314572800~4194304,385875968~4194304] 2024-09-13T00:52:15.774 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [37748736~8388608,79691776~4194304,92274688~12582912,155189248~29360128,192937984~25165824,247463936~8388608,260046848~20971520,285212672~8388608,301989888~8388608,314572800~8388608,385875968~4194304] 2024-09-13T00:52:15.774 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 11 diff [12582912~4194304,29360128~16777216,79691776~4194304,92274688~8388608,125829120~8388608,155189248~8388608,167772160~16777216,192937984~16777216,213909504~4194304,247463936~8388608,260046848~16777216,285212672~8388608,301989888~4194304,314572800~4194304,327155712~8388608,348127232~4194304,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.774 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~4194304,29360128~16777216,79691776~4194304,92274688~8388608,125829120~8388608,155189248~8388608,167772160~16777216,192937984~16777216,213909504~4194304,247463936~8388608,260046848~16777216,285212672~8388608,301989888~4194304,314572800~4194304,327155712~8388608,348127232~4194304,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:15 smithi077 bash[83127]: cluster 2024-09-13T00:52:14.412556+0000 mon.a (mon.0) 1478 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2024-09-13T00:52:15.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:15 smithi077 bash[80681]: cluster 2024-09-13T00:52:14.412556+0000 mon.a (mon.0) 1478 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2024-09-13T00:52:15.779 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~8388608,29360128~16777216,79691776~4194304,92274688~12582912,125829120~8388608,155189248~33554432,192937984~25165824,247463936~8388608,260046848~20971520,285212672~8388608,301989888~8388608,314572800~8388608,327155712~8388608,348127232~8388608,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.779 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 12 diff [12582912~4194304,29360128~25165824,71303168~16777216,92274688~8388608,125829120~8388608,155189248~8388608,167772160~20971520,192937984~16777216,213909504~4194304,247463936~8388608,260046848~20971520,285212672~8388608,301989888~16777216,327155712~8388608,348127232~8388608,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.779 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~4194304,29360128~25165824,71303168~16777216,92274688~8388608,125829120~8388608,155189248~8388608,167772160~20971520,192937984~16777216,213909504~4194304,247463936~8388608,260046848~20971520,285212672~8388608,301989888~16777216,327155712~8388608,348127232~8388608,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.785 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~8388608,29360128~25165824,71303168~33554432,125829120~8388608,155189248~62914560,247463936~8388608,260046848~33554432,301989888~20971520,327155712~8388608,348127232~8388608,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.785 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 13 diff [12582912~41943040,58720256~8388608,71303168~16777216,92274688~8388608,104857600~4194304,125829120~8388608,138412032~4194304,155189248~8388608,167772160~20971520,192937984~16777216,213909504~4194304,247463936~8388608,260046848~20971520,285212672~8388608,301989888~16777216,322961408~12582912,348127232~8388608,369098752~4194304,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.785 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~41943040,58720256~8388608,71303168~16777216,92274688~8388608,104857600~4194304,125829120~8388608,138412032~4194304,155189248~8388608,167772160~20971520,192937984~16777216,213909504~4194304,247463936~8388608,260046848~20971520,285212672~8388608,301989888~16777216,322961408~12582912,348127232~8388608,369098752~4194304,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.792 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~41943040,58720256~50331648,125829120~8388608,138412032~4194304,155189248~62914560,247463936~8388608,260046848~33554432,301989888~33554432,348127232~8388608,369098752~8388608,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.792 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 14 diff [12582912~37748736,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,260046848~20971520,285212672~8388608,301989888~16777216,322961408~12582912,348127232~8388608,369098752~4194304,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.792 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~37748736,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,260046848~20971520,285212672~8388608,301989888~16777216,322961408~12582912,348127232~8388608,369098752~4194304,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.801 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~96468992,125829120~8388608,138412032~4194304,155189248~62914560,247463936~8388608,260046848~33554432,301989888~33554432,348127232~8388608,369098752~8388608,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.801 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 15 diff [12582912~37748736,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~4194304,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,260046848~20971520,285212672~8388608,301989888~16777216,322961408~12582912,348127232~8388608,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.801 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~37748736,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~4194304,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,260046848~20971520,285212672~8388608,301989888~16777216,322961408~12582912,348127232~8388608,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.810 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~96468992,113246208~8388608,125829120~8388608,138412032~4194304,155189248~62914560,247463936~8388608,260046848~33554432,301989888~33554432,348127232~8388608,369098752~29360128] 2024-09-13T00:52:15.810 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 16 diff [12582912~37748736,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~8388608,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~33554432,285212672~8388608,301989888~41943040,348127232~8388608,360710144~4194304,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.810 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~37748736,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~8388608,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~33554432,285212672~8388608,301989888~41943040,348127232~8388608,360710144~4194304,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.820 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~96468992,113246208~8388608,125829120~8388608,138412032~4194304,155189248~62914560,247463936~46137344,301989888~54525952,360710144~37748736] 2024-09-13T00:52:15.820 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 17 diff [4194304~46137344,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~20971520,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~33554432,285212672~8388608,301989888~29360128,335544320~29360128,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.820 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~46137344,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~20971520,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~33554432,285212672~8388608,301989888~29360128,335544320~29360128,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.831 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~104857600,113246208~20971520,138412032~4194304,155189248~62914560,247463936~46137344,301989888~96468992] 2024-09-13T00:52:15.831 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 18 diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~4194304,113246208~20971520,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,239075328~41943040,285212672~8388608,301989888~29360128,335544320~37748736,377487360~20971520] 2024-09-13T00:52:15.831 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~4194304,113246208~20971520,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,239075328~41943040,285212672~8388608,301989888~29360128,335544320~37748736,377487360~20971520] 2024-09-13T00:52:15.842 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~104857600,113246208~20971520,138412032~4194304,155189248~62914560,239075328~54525952,301989888~96468992] 2024-09-13T00:52:15.842 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to 19 diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:15.842 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:15.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:15 smithi088 bash[68960]: cluster 2024-09-13T00:52:14.412556+0000 mon.a (mon.0) 1478 : cluster [DBG] osdmap e931: 8 total, 8 up, 8 in 2024-09-13T00:52:15.855 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~54525952,297795584~100663296] 2024-09-13T00:52:15.855 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 10 diff [92274688~8388608,155189248~8388608,167772160~12582912,201326592~8388608,213909504~4194304,247463936~4194304,272629760~8388608,314572800~4194304,385875968~4194304] 2024-09-13T00:52:15.855 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [92274688~8388608,155189248~8388608,167772160~12582912,201326592~8388608,213909504~4194304,247463936~4194304,272629760~8388608,314572800~4194304,385875968~4194304] 2024-09-13T00:52:15.858 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [92274688~12582912,155189248~29360128,201326592~16777216,247463936~8388608,272629760~8388608,314572800~8388608,385875968~4194304] 2024-09-13T00:52:15.858 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 11 diff [12582912~4194304,29360128~12582912,92274688~8388608,125829120~8388608,155189248~8388608,167772160~16777216,201326592~8388608,213909504~4194304,247463936~8388608,272629760~8388608,314572800~4194304,327155712~8388608,348127232~4194304,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.858 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~4194304,29360128~12582912,92274688~8388608,125829120~8388608,155189248~8388608,167772160~16777216,201326592~8388608,213909504~4194304,247463936~8388608,272629760~8388608,314572800~4194304,327155712~8388608,348127232~4194304,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.862 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~8388608,29360128~16777216,92274688~12582912,125829120~8388608,155189248~33554432,201326592~16777216,247463936~8388608,272629760~8388608,314572800~8388608,327155712~8388608,348127232~8388608,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.862 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 12 diff [12582912~4194304,29360128~12582912,46137344~8388608,71303168~8388608,83886080~4194304,92274688~8388608,125829120~8388608,155189248~8388608,167772160~20971520,201326592~8388608,213909504~4194304,247463936~8388608,272629760~12582912,306184192~12582912,327155712~8388608,348127232~8388608,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.862 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~4194304,29360128~12582912,46137344~8388608,71303168~8388608,83886080~4194304,92274688~8388608,125829120~8388608,155189248~8388608,167772160~20971520,201326592~8388608,213909504~4194304,247463936~8388608,272629760~12582912,306184192~12582912,327155712~8388608,348127232~8388608,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.867 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~8388608,29360128~25165824,71303168~33554432,125829120~8388608,155189248~37748736,201326592~16777216,247463936~8388608,272629760~12582912,306184192~16777216,327155712~8388608,348127232~8388608,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.868 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 13 diff [12582912~29360128,46137344~8388608,58720256~8388608,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,125829120~8388608,138412032~4194304,155189248~8388608,167772160~20971520,201326592~8388608,213909504~4194304,247463936~8388608,272629760~12582912,306184192~12582912,322961408~12582912,348127232~8388608,369098752~4194304,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.868 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~29360128,46137344~8388608,58720256~8388608,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,125829120~8388608,138412032~4194304,155189248~8388608,167772160~20971520,201326592~8388608,213909504~4194304,247463936~8388608,272629760~12582912,306184192~12582912,322961408~12582912,348127232~8388608,369098752~4194304,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.873 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~41943040,58720256~50331648,125829120~8388608,138412032~4194304,155189248~37748736,201326592~16777216,247463936~8388608,272629760~12582912,306184192~29360128,348127232~8388608,369098752~8388608,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.874 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 14 diff [12582912~37748736,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,272629760~12582912,306184192~12582912,322961408~12582912,348127232~8388608,369098752~4194304,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.874 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~37748736,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,272629760~12582912,306184192~12582912,322961408~12582912,348127232~8388608,369098752~4194304,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.881 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~96468992,125829120~8388608,138412032~4194304,155189248~62914560,247463936~8388608,272629760~12582912,306184192~29360128,348127232~8388608,369098752~8388608,381681664~8388608,394264576~4194304] 2024-09-13T00:52:15.881 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 15 diff [12582912~37748736,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,113246208~4194304,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,264241152~4194304,272629760~12582912,301989888~16777216,322961408~12582912,348127232~8388608,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.881 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~37748736,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,113246208~4194304,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~8388608,264241152~4194304,272629760~12582912,301989888~16777216,322961408~12582912,348127232~8388608,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.889 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~96468992,113246208~8388608,125829120~8388608,138412032~4194304,155189248~62914560,247463936~8388608,264241152~4194304,272629760~12582912,301989888~33554432,348127232~8388608,369098752~29360128] 2024-09-13T00:52:15.889 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 16 diff [12582912~37748736,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,113246208~8388608,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~12582912,264241152~4194304,272629760~12582912,301989888~41943040,348127232~8388608,360710144~4194304,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.889 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~37748736,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,113246208~8388608,125829120~8388608,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~12582912,264241152~4194304,272629760~12582912,301989888~41943040,348127232~8388608,360710144~4194304,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.898 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~96468992,113246208~8388608,125829120~8388608,138412032~4194304,155189248~62914560,247463936~12582912,264241152~4194304,272629760~12582912,301989888~54525952,360710144~37748736] 2024-09-13T00:52:15.898 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 17 diff [4194304~46137344,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~20971520,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~12582912,264241152~4194304,272629760~12582912,301989888~29360128,335544320~29360128,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.898 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~46137344,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~20971520,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,247463936~12582912,264241152~4194304,272629760~12582912,301989888~29360128,335544320~29360128,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.908 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~104857600,113246208~20971520,138412032~4194304,155189248~62914560,247463936~12582912,264241152~4194304,272629760~12582912,301989888~96468992] 2024-09-13T00:52:15.908 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 18 diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~4194304,113246208~20971520,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,239075328~20971520,264241152~4194304,272629760~12582912,301989888~29360128,335544320~37748736,377487360~20971520] 2024-09-13T00:52:15.908 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~4194304,113246208~20971520,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,239075328~20971520,264241152~4194304,272629760~12582912,301989888~29360128,335544320~37748736,377487360~20971520] 2024-09-13T00:52:15.919 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~104857600,113246208~20971520,138412032~4194304,155189248~62914560,239075328~20971520,264241152~4194304,272629760~12582912,301989888~96468992] 2024-09-13T00:52:15.920 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to 19 diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:15.920 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:15.932 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:15.932 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 11 diff [12582912~4194304,29360128~12582912,125829120~8388608,180355072~4194304,247463936~4194304,327155712~8388608,348127232~4194304,381681664~4194304,394264576~4194304] 2024-09-13T00:52:15.932 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~4194304,29360128~12582912,125829120~8388608,180355072~4194304,247463936~4194304,327155712~8388608,348127232~4194304,381681664~4194304,394264576~4194304] 2024-09-13T00:52:15.936 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~8388608,29360128~16777216,125829120~8388608,180355072~8388608,247463936~4194304,327155712~8388608,348127232~8388608,381681664~4194304,394264576~4194304] 2024-09-13T00:52:15.936 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 12 diff [12582912~4194304,29360128~12582912,46137344~8388608,71303168~8388608,83886080~4194304,125829120~8388608,171966464~16777216,247463936~4194304,272629760~8388608,306184192~8388608,327155712~8388608,348127232~8388608,381681664~4194304,394264576~4194304] 2024-09-13T00:52:15.936 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~4194304,29360128~12582912,46137344~8388608,71303168~8388608,83886080~4194304,125829120~8388608,171966464~16777216,247463936~4194304,272629760~8388608,306184192~8388608,327155712~8388608,348127232~8388608,381681664~4194304,394264576~4194304] 2024-09-13T00:52:15.940 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~8388608,29360128~25165824,71303168~20971520,125829120~8388608,171966464~20971520,247463936~4194304,272629760~12582912,306184192~12582912,327155712~8388608,348127232~8388608,381681664~4194304,394264576~4194304] 2024-09-13T00:52:15.940 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 13 diff [12582912~29360128,46137344~8388608,58720256~8388608,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,125829120~8388608,138412032~4194304,171966464~16777216,247463936~4194304,272629760~12582912,306184192~8388608,322961408~12582912,348127232~8388608,369098752~4194304,381681664~4194304,394264576~4194304] 2024-09-13T00:52:15.941 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~29360128,46137344~8388608,58720256~8388608,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,125829120~8388608,138412032~4194304,171966464~16777216,247463936~4194304,272629760~12582912,306184192~8388608,322961408~12582912,348127232~8388608,369098752~4194304,381681664~4194304,394264576~4194304] 2024-09-13T00:52:15.946 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~41943040,58720256~50331648,125829120~8388608,138412032~4194304,171966464~20971520,247463936~4194304,272629760~12582912,306184192~12582912,322961408~12582912,348127232~8388608,369098752~8388608,381681664~4194304,394264576~4194304] 2024-09-13T00:52:15.946 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 14 diff [12582912~37748736,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,125829120~8388608,138412032~4194304,163577856~25165824,192937984~8388608,247463936~4194304,272629760~12582912,306184192~8388608,322961408~12582912,348127232~8388608,369098752~4194304,381681664~4194304,394264576~4194304] 2024-09-13T00:52:15.946 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~37748736,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,125829120~8388608,138412032~4194304,163577856~25165824,192937984~8388608,247463936~4194304,272629760~12582912,306184192~8388608,322961408~12582912,348127232~8388608,369098752~4194304,381681664~4194304,394264576~4194304] 2024-09-13T00:52:15.953 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~96468992,125829120~8388608,138412032~4194304,163577856~37748736,247463936~4194304,272629760~12582912,306184192~12582912,322961408~12582912,348127232~8388608,369098752~8388608,381681664~4194304,394264576~4194304] 2024-09-13T00:52:15.953 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 15 diff [12582912~37748736,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,113246208~4194304,125829120~8388608,138412032~4194304,163577856~25165824,192937984~8388608,247463936~4194304,264241152~4194304,272629760~12582912,301989888~12582912,322961408~12582912,348127232~8388608,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.953 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~37748736,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,113246208~4194304,125829120~8388608,138412032~4194304,163577856~25165824,192937984~8388608,247463936~4194304,264241152~4194304,272629760~12582912,301989888~12582912,322961408~12582912,348127232~8388608,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.960 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~96468992,113246208~8388608,125829120~8388608,138412032~4194304,163577856~37748736,247463936~4194304,264241152~4194304,272629760~12582912,301989888~16777216,322961408~12582912,348127232~8388608,369098752~29360128] 2024-09-13T00:52:15.960 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 16 diff [12582912~37748736,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,113246208~8388608,125829120~8388608,138412032~4194304,163577856~25165824,192937984~12582912,247463936~12582912,264241152~4194304,272629760~12582912,301989888~41943040,348127232~8388608,360710144~4194304,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.960 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~37748736,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,113246208~8388608,125829120~8388608,138412032~4194304,163577856~25165824,192937984~12582912,247463936~12582912,264241152~4194304,272629760~12582912,301989888~41943040,348127232~8388608,360710144~4194304,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.968 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~96468992,113246208~8388608,125829120~8388608,138412032~4194304,163577856~41943040,247463936~12582912,264241152~4194304,272629760~12582912,301989888~54525952,360710144~37748736] 2024-09-13T00:52:15.969 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 17 diff [4194304~46137344,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~20971520,138412032~4194304,163577856~25165824,192937984~12582912,247463936~12582912,264241152~4194304,272629760~12582912,301989888~29360128,335544320~29360128,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.969 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~46137344,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~20971520,138412032~4194304,163577856~25165824,192937984~12582912,247463936~12582912,264241152~4194304,272629760~12582912,301989888~29360128,335544320~29360128,369098752~4194304,377487360~20971520] 2024-09-13T00:52:15.978 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~104857600,113246208~20971520,138412032~4194304,163577856~41943040,247463936~12582912,264241152~4194304,272629760~12582912,301989888~96468992] 2024-09-13T00:52:15.978 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 18 diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~4194304,113246208~20971520,138412032~4194304,163577856~25165824,192937984~12582912,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,301989888~29360128,335544320~37748736,377487360~20971520] 2024-09-13T00:52:15.978 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~4194304,113246208~20971520,138412032~4194304,163577856~25165824,192937984~12582912,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,301989888~29360128,335544320~37748736,377487360~20971520] 2024-09-13T00:52:15.988 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~104857600,113246208~20971520,138412032~4194304,163577856~41943040,239075328~20971520,264241152~4194304,272629760~12582912,301989888~96468992] 2024-09-13T00:52:15.988 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to 19 diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:15.988 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:16.000 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:16.000 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to 12 diff [46137344~8388608,71303168~8388608,83886080~4194304,171966464~16777216,272629760~8388608,306184192~8388608,352321536~4194304] 2024-09-13T00:52:16.000 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [46137344~8388608,71303168~8388608,83886080~4194304,171966464~16777216,272629760~8388608,306184192~8388608,352321536~4194304] 2024-09-13T00:52:16.003 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [46137344~8388608,71303168~20971520,171966464~20971520,272629760~12582912,306184192~12582912,352321536~4194304] 2024-09-13T00:52:16.003 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to 13 diff [12582912~16777216,46137344~8388608,58720256~8388608,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,138412032~4194304,171966464~16777216,272629760~12582912,306184192~8388608,322961408~8388608,352321536~4194304,369098752~4194304] 2024-09-13T00:52:16.003 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~16777216,46137344~8388608,58720256~8388608,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,138412032~4194304,171966464~16777216,272629760~12582912,306184192~8388608,322961408~8388608,352321536~4194304,369098752~4194304] 2024-09-13T00:52:16.009 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~20971520,46137344~8388608,58720256~50331648,138412032~4194304,171966464~20971520,272629760~12582912,306184192~12582912,322961408~12582912,352321536~4194304,369098752~8388608] 2024-09-13T00:52:16.009 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to 14 diff [12582912~25165824,41943040~8388608,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,138412032~4194304,163577856~25165824,192937984~8388608,272629760~12582912,306184192~8388608,322961408~8388608,352321536~4194304,369098752~4194304] 2024-09-13T00:52:16.009 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~25165824,41943040~8388608,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,138412032~4194304,163577856~25165824,192937984~8388608,272629760~12582912,306184192~8388608,322961408~8388608,352321536~4194304,369098752~4194304] 2024-09-13T00:52:16.016 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~25165824,41943040~67108864,138412032~4194304,163577856~37748736,272629760~12582912,306184192~12582912,322961408~12582912,352321536~4194304,369098752~8388608] 2024-09-13T00:52:16.016 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to 15 diff [12582912~25165824,41943040~8388608,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,113246208~4194304,138412032~4194304,163577856~25165824,192937984~8388608,264241152~4194304,272629760~12582912,301989888~12582912,322961408~8388608,352321536~4194304,369098752~4194304,377487360~16777216] 2024-09-13T00:52:16.016 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~25165824,41943040~8388608,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,113246208~4194304,138412032~4194304,163577856~25165824,192937984~8388608,264241152~4194304,272629760~12582912,301989888~12582912,322961408~8388608,352321536~4194304,369098752~4194304,377487360~16777216] 2024-09-13T00:52:16.021 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~25165824,41943040~67108864,113246208~8388608,138412032~4194304,163577856~37748736,264241152~4194304,272629760~12582912,301989888~16777216,322961408~12582912,352321536~4194304,369098752~25165824] 2024-09-13T00:52:16.021 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to 16 diff [12582912~37748736,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,113246208~8388608,138412032~4194304,163577856~25165824,192937984~12582912,251658240~8388608,264241152~4194304,272629760~12582912,301989888~29360128,335544320~8388608,352321536~4194304,360710144~4194304,369098752~4194304,377487360~16777216] 2024-09-13T00:52:16.021 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~37748736,54525952~12582912,71303168~8388608,83886080~4194304,92274688~8388608,104857600~4194304,113246208~8388608,138412032~4194304,163577856~25165824,192937984~12582912,251658240~8388608,264241152~4194304,272629760~12582912,301989888~29360128,335544320~8388608,352321536~4194304,360710144~4194304,369098752~4194304,377487360~16777216] 2024-09-13T00:52:16.029 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~96468992,113246208~8388608,138412032~4194304,163577856~41943040,251658240~8388608,264241152~4194304,272629760~12582912,301989888~46137344,352321536~4194304,360710144~33554432] 2024-09-13T00:52:16.029 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to 17 diff [4194304~46137344,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~16777216,138412032~4194304,163577856~25165824,192937984~12582912,251658240~8388608,264241152~4194304,272629760~12582912,301989888~29360128,335544320~12582912,352321536~12582912,369098752~4194304,377487360~16777216] 2024-09-13T00:52:16.029 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~46137344,54525952~12582912,71303168~16777216,92274688~8388608,104857600~4194304,113246208~16777216,138412032~4194304,163577856~25165824,192937984~12582912,251658240~8388608,264241152~4194304,272629760~12582912,301989888~29360128,335544320~12582912,352321536~12582912,369098752~4194304,377487360~16777216] 2024-09-13T00:52:16.038 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~104857600,113246208~16777216,138412032~4194304,163577856~41943040,251658240~8388608,264241152~4194304,272629760~12582912,301989888~92274688] 2024-09-13T00:52:16.038 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to 18 diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~4194304,113246208~16777216,138412032~4194304,163577856~25165824,192937984~12582912,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,301989888~29360128,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:16.038 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~4194304,113246208~16777216,138412032~4194304,163577856~25165824,192937984~12582912,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,301989888~29360128,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:16.046 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~104857600,113246208~16777216,138412032~4194304,163577856~41943040,239075328~20971520,264241152~4194304,272629760~12582912,301989888~92274688] 2024-09-13T00:52:16.046 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to 19 diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:16.047 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:16.057 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~125829120,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~96468992] 2024-09-13T00:52:16.058 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to 13 diff [12582912~16777216,58720256~8388608,88080384~12582912,104857600~4194304,138412032~4194304,276824064~4194304,322961408~8388608,369098752~4194304] 2024-09-13T00:52:16.058 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~16777216,58720256~8388608,88080384~12582912,104857600~4194304,138412032~4194304,276824064~4194304,322961408~8388608,369098752~4194304] 2024-09-13T00:52:16.061 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~20971520,58720256~12582912,88080384~20971520,138412032~4194304,276824064~8388608,322961408~12582912,369098752~8388608] 2024-09-13T00:52:16.061 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to 14 diff [12582912~25165824,41943040~4194304,54525952~12582912,88080384~12582912,104857600~4194304,138412032~4194304,163577856~8388608,176160768~4194304,192937984~8388608,272629760~8388608,322961408~8388608,369098752~4194304] 2024-09-13T00:52:16.061 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~25165824,41943040~4194304,54525952~12582912,88080384~12582912,104857600~4194304,138412032~4194304,163577856~8388608,176160768~4194304,192937984~8388608,272629760~8388608,322961408~8388608,369098752~4194304] 2024-09-13T00:52:16.066 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~25165824,41943040~8388608,54525952~16777216,88080384~20971520,138412032~4194304,163577856~8388608,176160768~8388608,192937984~8388608,272629760~12582912,322961408~12582912,369098752~8388608] 2024-09-13T00:52:16.066 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to 15 diff [12582912~25165824,41943040~4194304,54525952~12582912,88080384~12582912,104857600~4194304,113246208~4194304,138412032~4194304,163577856~25165824,192937984~8388608,264241152~4194304,272629760~8388608,301989888~4194304,322961408~8388608,352321536~4194304,369098752~4194304,377487360~16777216] 2024-09-13T00:52:16.066 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~25165824,41943040~4194304,54525952~12582912,88080384~12582912,104857600~4194304,113246208~4194304,138412032~4194304,163577856~25165824,192937984~8388608,264241152~4194304,272629760~8388608,301989888~4194304,322961408~8388608,352321536~4194304,369098752~4194304,377487360~16777216] 2024-09-13T00:52:16.071 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~25165824,41943040~8388608,54525952~16777216,88080384~20971520,113246208~8388608,138412032~4194304,163577856~25165824,192937984~8388608,264241152~4194304,272629760~12582912,301989888~8388608,322961408~12582912,352321536~4194304,369098752~25165824] 2024-09-13T00:52:16.071 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to 16 diff [12582912~33554432,54525952~12582912,88080384~12582912,104857600~4194304,113246208~8388608,138412032~4194304,163577856~25165824,192937984~12582912,251658240~8388608,264241152~4194304,272629760~8388608,301989888~8388608,314572800~16777216,335544320~8388608,352321536~4194304,360710144~4194304,369098752~4194304,377487360~16777216] 2024-09-13T00:52:16.071 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~33554432,54525952~12582912,88080384~12582912,104857600~4194304,113246208~8388608,138412032~4194304,163577856~25165824,192937984~12582912,251658240~8388608,264241152~4194304,272629760~8388608,301989888~8388608,314572800~16777216,335544320~8388608,352321536~4194304,360710144~4194304,369098752~4194304,377487360~16777216] 2024-09-13T00:52:16.078 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~37748736,54525952~16777216,88080384~20971520,113246208~8388608,138412032~4194304,163577856~25165824,192937984~12582912,251658240~8388608,264241152~4194304,272629760~12582912,301989888~8388608,314572800~33554432,352321536~4194304,360710144~33554432] 2024-09-13T00:52:16.078 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to 17 diff [4194304~41943040,54525952~12582912,71303168~4194304,88080384~12582912,104857600~4194304,113246208~16777216,138412032~4194304,163577856~25165824,192937984~12582912,251658240~8388608,264241152~4194304,272629760~8388608,301989888~8388608,314572800~16777216,335544320~12582912,352321536~12582912,369098752~4194304,377487360~16777216] 2024-09-13T00:52:16.078 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~41943040,54525952~12582912,71303168~4194304,88080384~12582912,104857600~4194304,113246208~16777216,138412032~4194304,163577856~25165824,192937984~12582912,251658240~8388608,264241152~4194304,272629760~8388608,301989888~8388608,314572800~16777216,335544320~12582912,352321536~12582912,369098752~4194304,377487360~16777216] 2024-09-13T00:52:16.085 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~46137344,54525952~20971520,88080384~20971520,113246208~16777216,138412032~4194304,163577856~25165824,192937984~12582912,251658240~8388608,264241152~4194304,272629760~12582912,301989888~8388608,314572800~79691776] 2024-09-13T00:52:16.085 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to 18 diff [4194304~62914560,71303168~4194304,83886080~16777216,104857600~4194304,113246208~16777216,138412032~4194304,163577856~25165824,192937984~12582912,239075328~8388608,251658240~8388608,264241152~4194304,272629760~8388608,301989888~8388608,314572800~16777216,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:16.085 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~4194304,83886080~16777216,104857600~4194304,113246208~16777216,138412032~4194304,163577856~25165824,192937984~12582912,239075328~8388608,251658240~8388608,264241152~4194304,272629760~8388608,301989888~8388608,314572800~16777216,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:16.094 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~71303168,83886080~25165824,113246208~16777216,138412032~4194304,163577856~25165824,192937984~12582912,239075328~20971520,264241152~4194304,272629760~12582912,301989888~8388608,314572800~79691776] 2024-09-13T00:52:16.094 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to 19 diff [4194304~62914560,71303168~4194304,83886080~16777216,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~16777216,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:16.094 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~4194304,83886080~16777216,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~16777216,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:16.104 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~71303168,83886080~46137344,138412032~4194304,163577856~25165824,192937984~12582912,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~12582912,314572800~79691776] 2024-09-13T00:52:16.104 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to 14 diff [12582912~8388608,25165824~8388608,41943040~4194304,54525952~8388608,163577856~8388608,176160768~4194304,192937984~8388608,272629760~4194304] 2024-09-13T00:52:16.104 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~8388608,25165824~8388608,41943040~4194304,54525952~8388608,163577856~8388608,176160768~4194304,192937984~8388608,272629760~4194304] 2024-09-13T00:52:16.108 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~8388608,25165824~12582912,41943040~8388608,54525952~12582912,163577856~8388608,176160768~8388608,192937984~8388608,272629760~4194304] 2024-09-13T00:52:16.108 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to 15 diff [12582912~8388608,25165824~8388608,41943040~4194304,54525952~8388608,113246208~4194304,163577856~25165824,192937984~8388608,264241152~4194304,272629760~4194304,301989888~4194304,352321536~4194304,377487360~16777216] 2024-09-13T00:52:16.108 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~8388608,25165824~8388608,41943040~4194304,54525952~8388608,113246208~4194304,163577856~25165824,192937984~8388608,264241152~4194304,272629760~4194304,301989888~4194304,352321536~4194304,377487360~16777216] 2024-09-13T00:52:16.112 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~8388608,25165824~12582912,41943040~8388608,54525952~12582912,113246208~8388608,163577856~25165824,192937984~8388608,264241152~4194304,272629760~4194304,301989888~8388608,352321536~4194304,377487360~16777216] 2024-09-13T00:52:16.112 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to 16 diff [12582912~8388608,25165824~8388608,37748736~8388608,54525952~8388608,113246208~8388608,163577856~25165824,192937984~12582912,251658240~8388608,264241152~4194304,272629760~4194304,301989888~8388608,314572800~8388608,335544320~8388608,352321536~4194304,360710144~4194304,377487360~16777216] 2024-09-13T00:52:16.113 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~8388608,25165824~8388608,37748736~8388608,54525952~8388608,113246208~8388608,163577856~25165824,192937984~12582912,251658240~8388608,264241152~4194304,272629760~4194304,301989888~8388608,314572800~8388608,335544320~8388608,352321536~4194304,360710144~4194304,377487360~16777216] 2024-09-13T00:52:16.118 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~8388608,25165824~25165824,54525952~12582912,113246208~8388608,163577856~25165824,192937984~12582912,251658240~8388608,264241152~4194304,272629760~4194304,301989888~8388608,314572800~8388608,335544320~12582912,352321536~4194304,360710144~8388608,377487360~16777216] 2024-09-13T00:52:16.118 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to 17 diff [4194304~16777216,25165824~8388608,37748736~8388608,54525952~8388608,71303168~4194304,113246208~16777216,138412032~4194304,163577856~25165824,192937984~12582912,251658240~8388608,264241152~4194304,272629760~4194304,301989888~8388608,314572800~12582912,335544320~12582912,352321536~12582912,377487360~16777216] 2024-09-13T00:52:16.118 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~8388608,54525952~8388608,71303168~4194304,113246208~16777216,138412032~4194304,163577856~25165824,192937984~12582912,251658240~8388608,264241152~4194304,272629760~4194304,301989888~8388608,314572800~12582912,335544320~12582912,352321536~12582912,377487360~16777216] 2024-09-13T00:52:16.124 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~16777216,25165824~25165824,54525952~12582912,71303168~4194304,113246208~16777216,138412032~4194304,163577856~25165824,192937984~12582912,251658240~8388608,264241152~4194304,272629760~4194304,301989888~8388608,314572800~16777216,335544320~33554432,377487360~16777216] 2024-09-13T00:52:16.125 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to 18 diff [4194304~16777216,25165824~12582912,41943040~20971520,71303168~4194304,83886080~4194304,113246208~16777216,138412032~4194304,163577856~25165824,192937984~12582912,239075328~8388608,251658240~8388608,264241152~4194304,272629760~4194304,301989888~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:16.125 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,25165824~12582912,41943040~20971520,71303168~4194304,83886080~4194304,113246208~16777216,138412032~4194304,163577856~25165824,192937984~12582912,239075328~8388608,251658240~8388608,264241152~4194304,272629760~4194304,301989888~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:16.132 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~62914560,71303168~4194304,83886080~4194304,113246208~16777216,138412032~4194304,163577856~25165824,192937984~12582912,239075328~20971520,264241152~4194304,272629760~4194304,301989888~8388608,314572800~16777216,335544320~37748736,377487360~16777216] 2024-09-13T00:52:16.132 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to 19 diff [4194304~58720256,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:16.132 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~58720256,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:16.141 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~62914560,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~12582912,314572800~16777216,335544320~37748736,377487360~16777216] 2024-09-13T00:52:16.141 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to 15 diff [113246208~4194304,167772160~8388608,180355072~8388608,264241152~4194304,301989888~4194304,352321536~4194304,377487360~16777216] 2024-09-13T00:52:16.141 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [113246208~4194304,167772160~8388608,180355072~8388608,264241152~4194304,301989888~4194304,352321536~4194304,377487360~16777216] 2024-09-13T00:52:16.145 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [113246208~8388608,167772160~8388608,180355072~8388608,264241152~4194304,301989888~8388608,352321536~4194304,377487360~16777216] 2024-09-13T00:52:16.145 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to 16 diff [37748736~4194304,113246208~8388608,167772160~8388608,180355072~8388608,201326592~4194304,251658240~8388608,264241152~4194304,301989888~8388608,314572800~8388608,335544320~8388608,352321536~4194304,360710144~4194304,377487360~16777216] 2024-09-13T00:52:16.145 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [37748736~4194304,113246208~8388608,167772160~8388608,180355072~8388608,201326592~4194304,251658240~8388608,264241152~4194304,301989888~8388608,314572800~8388608,335544320~8388608,352321536~4194304,360710144~4194304,377487360~16777216] 2024-09-13T00:52:16.149 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [37748736~4194304,113246208~8388608,167772160~8388608,180355072~8388608,201326592~4194304,251658240~8388608,264241152~4194304,301989888~8388608,314572800~8388608,335544320~12582912,352321536~4194304,360710144~8388608,377487360~16777216] 2024-09-13T00:52:16.149 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to 17 diff [4194304~8388608,37748736~4194304,54525952~4194304,71303168~4194304,113246208~16777216,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,251658240~8388608,264241152~4194304,301989888~8388608,314572800~12582912,335544320~12582912,352321536~12582912,377487360~16777216] 2024-09-13T00:52:16.149 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~8388608,37748736~4194304,54525952~4194304,71303168~4194304,113246208~16777216,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,251658240~8388608,264241152~4194304,301989888~8388608,314572800~12582912,335544320~12582912,352321536~12582912,377487360~16777216] 2024-09-13T00:52:16.154 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~8388608,37748736~4194304,54525952~8388608,71303168~4194304,113246208~16777216,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,251658240~8388608,264241152~4194304,301989888~8388608,314572800~16777216,335544320~33554432,377487360~16777216] 2024-09-13T00:52:16.154 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to 18 diff [4194304~16777216,33554432~8388608,46137344~12582912,71303168~4194304,83886080~4194304,113246208~16777216,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,239075328~8388608,251658240~8388608,264241152~4194304,301989888~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:16.155 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,33554432~8388608,46137344~12582912,71303168~4194304,83886080~4194304,113246208~16777216,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,239075328~8388608,251658240~8388608,264241152~4194304,301989888~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:16.161 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~20971520,33554432~8388608,46137344~16777216,71303168~4194304,83886080~4194304,113246208~16777216,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,239075328~20971520,264241152~4194304,301989888~8388608,314572800~16777216,335544320~37748736,377487360~16777216] 2024-09-13T00:52:16.161 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to 19 diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:16.161 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:16.169 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~20971520,33554432~29360128,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,222298112~12582912,239075328~20971520,264241152~4194304,276824064~8388608,297795584~12582912,314572800~16777216,335544320~37748736,377487360~16777216] 2024-09-13T00:52:16.169 INFO:tasks.workunit.client.0.smithi077.stdout:from 15 to 16 diff [37748736~4194304,117440512~4194304,201326592~4194304,251658240~8388608,301989888~8388608,314572800~8388608,335544320~8388608,360710144~4194304] 2024-09-13T00:52:16.169 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [37748736~4194304,117440512~4194304,201326592~4194304,251658240~8388608,301989888~8388608,314572800~8388608,335544320~8388608,360710144~4194304] 2024-09-13T00:52:16.173 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [37748736~4194304,117440512~4194304,201326592~4194304,251658240~8388608,301989888~8388608,314572800~8388608,335544320~12582912,360710144~8388608] 2024-09-13T00:52:16.173 INFO:tasks.workunit.client.0.smithi077.stdout:from 15 to 17 diff [4194304~8388608,37748736~4194304,54525952~4194304,71303168~4194304,113246208~16777216,138412032~4194304,201326592~4194304,251658240~8388608,301989888~8388608,314572800~12582912,335544320~12582912,356515840~8388608,385875968~4194304] 2024-09-13T00:52:16.173 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~8388608,37748736~4194304,54525952~4194304,71303168~4194304,113246208~16777216,138412032~4194304,201326592~4194304,251658240~8388608,301989888~8388608,314572800~12582912,335544320~12582912,356515840~8388608,385875968~4194304] 2024-09-13T00:52:16.177 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~8388608,37748736~4194304,54525952~8388608,71303168~4194304,113246208~16777216,138412032~4194304,201326592~4194304,251658240~8388608,301989888~8388608,314572800~16777216,335544320~16777216,356515840~12582912,385875968~8388608] 2024-09-13T00:52:16.178 INFO:tasks.workunit.client.0.smithi077.stdout:from 15 to 18 diff [4194304~16777216,33554432~8388608,46137344~12582912,71303168~4194304,83886080~4194304,113246208~16777216,138412032~4194304,201326592~4194304,239075328~8388608,251658240~8388608,301989888~8388608,314572800~12582912,335544320~12582912,356515840~12582912,385875968~4194304] 2024-09-13T00:52:16.178 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,33554432~8388608,46137344~12582912,71303168~4194304,83886080~4194304,113246208~16777216,138412032~4194304,201326592~4194304,239075328~8388608,251658240~8388608,301989888~8388608,314572800~12582912,335544320~12582912,356515840~12582912,385875968~4194304] 2024-09-13T00:52:16.182 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~20971520,33554432~8388608,46137344~16777216,71303168~4194304,83886080~4194304,113246208~16777216,138412032~4194304,201326592~4194304,239075328~20971520,301989888~8388608,314572800~16777216,335544320~16777216,356515840~16777216,385875968~8388608] 2024-09-13T00:52:16.182 INFO:tasks.workunit.client.0.smithi077.stdout:from 15 to 19 diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,356515840~12582912,385875968~4194304] 2024-09-13T00:52:16.182 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,356515840~12582912,385875968~4194304] 2024-09-13T00:52:16.188 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~20971520,33554432~29360128,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,201326592~4194304,222298112~12582912,239075328~20971520,264241152~4194304,276824064~8388608,297795584~12582912,314572800~16777216,335544320~16777216,356515840~16777216,385875968~8388608] 2024-09-13T00:52:16.188 INFO:tasks.workunit.client.0.smithi077.stdout:from 16 to 17 diff [4194304~8388608,54525952~4194304,71303168~4194304,113246208~4194304,121634816~8388608,138412032~4194304,318767104~8388608,339738624~8388608,356515840~8388608,385875968~4194304] 2024-09-13T00:52:16.188 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~8388608,54525952~4194304,71303168~4194304,113246208~4194304,121634816~8388608,138412032~4194304,318767104~8388608,339738624~8388608,356515840~8388608,385875968~4194304] 2024-09-13T00:52:16.191 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~8388608,54525952~8388608,71303168~4194304,113246208~4194304,121634816~8388608,138412032~4194304,318767104~12582912,339738624~12582912,356515840~12582912,385875968~8388608] 2024-09-13T00:52:16.191 INFO:tasks.workunit.client.0.smithi077.stdout:from 16 to 18 diff [4194304~16777216,33554432~8388608,46137344~12582912,71303168~4194304,83886080~4194304,113246208~16777216,138412032~4194304,239075328~8388608,318767104~8388608,339738624~8388608,356515840~12582912,385875968~4194304] 2024-09-13T00:52:16.191 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,33554432~8388608,46137344~12582912,71303168~4194304,83886080~4194304,113246208~16777216,138412032~4194304,239075328~8388608,318767104~8388608,339738624~8388608,356515840~12582912,385875968~4194304] 2024-09-13T00:52:16.195 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~20971520,33554432~8388608,46137344~16777216,71303168~4194304,83886080~4194304,113246208~16777216,138412032~4194304,239075328~12582912,318767104~12582912,339738624~12582912,356515840~16777216,385875968~8388608] 2024-09-13T00:52:16.195 INFO:tasks.workunit.client.0.smithi077.stdout:from 16 to 19 diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,222298112~8388608,239075328~8388608,264241152~4194304,276824064~8388608,297795584~8388608,318767104~8388608,339738624~8388608,356515840~12582912,385875968~4194304] 2024-09-13T00:52:16.195 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,222298112~8388608,239075328~8388608,264241152~4194304,276824064~8388608,297795584~8388608,318767104~8388608,339738624~8388608,356515840~12582912,385875968~4194304] 2024-09-13T00:52:16.200 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~20971520,33554432~29360128,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,222298112~12582912,239075328~12582912,264241152~4194304,276824064~8388608,297795584~12582912,318767104~12582912,339738624~12582912,356515840~16777216,385875968~8388608] 2024-09-13T00:52:16.200 INFO:tasks.workunit.client.0.smithi077.stdout:from 17 to 18 diff [12582912~8388608,33554432~8388608,46137344~8388608,83886080~4194304,113246208~8388608,239075328~8388608,356515840~12582912] 2024-09-13T00:52:16.200 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~8388608,33554432~8388608,46137344~8388608,83886080~4194304,113246208~8388608,239075328~8388608,356515840~12582912] 2024-09-13T00:52:16.203 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~12582912,33554432~8388608,46137344~12582912,83886080~4194304,113246208~12582912,239075328~12582912,356515840~16777216] 2024-09-13T00:52:16.203 INFO:tasks.workunit.client.0.smithi077.stdout:from 17 to 19 diff [12582912~8388608,33554432~20971520,83886080~4194304,104857600~16777216,184549376~4194304,222298112~8388608,239075328~8388608,264241152~4194304,276824064~8388608,297795584~8388608,356515840~12582912] 2024-09-13T00:52:16.203 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~8388608,33554432~20971520,83886080~4194304,104857600~16777216,184549376~4194304,222298112~8388608,239075328~8388608,264241152~4194304,276824064~8388608,297795584~8388608,356515840~12582912] 2024-09-13T00:52:16.207 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~12582912,33554432~25165824,83886080~4194304,104857600~20971520,184549376~4194304,222298112~12582912,239075328~12582912,264241152~4194304,276824064~8388608,297795584~12582912,356515840~16777216] 2024-09-13T00:52:16.207 INFO:tasks.workunit.client.0.smithi077.stdout:from 18 to 19 diff [12582912~8388608,37748736~8388608,104857600~16777216,184549376~4194304,222298112~8388608,264241152~4194304,276824064~8388608,297795584~8388608] 2024-09-13T00:52:16.207 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~8388608,37748736~8388608,104857600~16777216,184549376~4194304,222298112~8388608,264241152~4194304,276824064~8388608,297795584~8388608] 2024-09-13T00:52:16.210 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~12582912,37748736~12582912,104857600~20971520,184549376~4194304,222298112~12582912,264241152~4194304,276824064~8388608,297795584~12582912] 2024-09-13T00:52:16.458 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:16.458 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:16.505 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:16.505 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:16.505 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:16.522 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:16.523 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:16.523 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:16.540 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:16.540 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:16.540 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:16.558 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:16.558 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:16.558 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:16.574 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:16.574 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:16.574 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:16.589 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:16.589 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:16.589 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:16.603 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:16.603 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:16.603 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:16.616 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:16.617 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:16.617 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:16.629 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~54525952,297795584~100663296] 2024-09-13T00:52:16.629 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:16.629 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:16.640 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:16.640 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:16.640 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:16.650 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:16.651 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:16.651 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:16.660 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~125829120,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~96468992] 2024-09-13T00:52:16.660 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to HEAD diff [4194304~62914560,71303168~4194304,83886080~16777216,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~16777216,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:16.660 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~4194304,83886080~16777216,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~16777216,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:16.669 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~71303168,83886080~46137344,138412032~4194304,163577856~25165824,192937984~12582912,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~12582912,314572800~79691776] 2024-09-13T00:52:16.669 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to HEAD diff [4194304~58720256,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:16.669 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~58720256,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:16.677 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~62914560,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~12582912,314572800~16777216,335544320~37748736,377487360~16777216] 2024-09-13T00:52:16.677 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to HEAD diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:16.677 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:16.683 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~20971520,33554432~29360128,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,222298112~12582912,239075328~20971520,264241152~4194304,276824064~8388608,297795584~12582912,314572800~16777216,335544320~37748736,377487360~16777216] 2024-09-13T00:52:16.683 INFO:tasks.workunit.client.0.smithi077.stdout:from 15 to HEAD diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,356515840~12582912,385875968~4194304] 2024-09-13T00:52:16.683 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,356515840~12582912,385875968~4194304] 2024-09-13T00:52:16.688 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~20971520,33554432~29360128,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,201326592~4194304,222298112~12582912,239075328~20971520,264241152~4194304,276824064~8388608,297795584~12582912,314572800~16777216,335544320~16777216,356515840~16777216,385875968~8388608] 2024-09-13T00:52:16.688 INFO:tasks.workunit.client.0.smithi077.stdout:from 16 to HEAD diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,222298112~8388608,239075328~8388608,264241152~4194304,276824064~8388608,297795584~8388608,318767104~8388608,339738624~8388608,356515840~12582912,385875968~4194304] 2024-09-13T00:52:16.688 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,222298112~8388608,239075328~8388608,264241152~4194304,276824064~8388608,297795584~8388608,318767104~8388608,339738624~8388608,356515840~12582912,385875968~4194304] 2024-09-13T00:52:16.692 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~20971520,33554432~29360128,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,222298112~12582912,239075328~12582912,264241152~4194304,276824064~8388608,297795584~12582912,318767104~12582912,339738624~12582912,356515840~16777216,385875968~8388608] 2024-09-13T00:52:16.692 INFO:tasks.workunit.client.0.smithi077.stdout:from 17 to HEAD diff [12582912~8388608,33554432~20971520,83886080~4194304,104857600~16777216,184549376~4194304,222298112~8388608,239075328~8388608,264241152~4194304,276824064~8388608,297795584~8388608,356515840~12582912] 2024-09-13T00:52:16.693 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [12582912~8388608,33554432~20971520,83886080~4194304,104857600~16777216,184549376~4194304,222298112~8388608,239075328~8388608,264241152~4194304,276824064~8388608,297795584~8388608,356515840~12582912] 2024-09-13T00:52:16.695 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [12582912~12582912,33554432~25165824,83886080~4194304,104857600~20971520,184549376~4194304,222298112~12582912,239075328~12582912,264241152~4194304,276824064~8388608,297795584~12582912,356515840~16777216] 2024-09-13T00:52:16.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:16 smithi077 bash[83127]: cluster 2024-09-13T00:52:14.853591+0000 mgr.y (mgr.25304) 491 : cluster [DBG] pgmap v1042: 105 pgs: 105 active+clean; 811 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 212 KiB/s rd, 47 MiB/s wr, 331 op/s 2024-09-13T00:52:16.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:16 smithi077 bash[80681]: cluster 2024-09-13T00:52:14.853591+0000 mgr.y (mgr.25304) 491 : cluster [DBG] pgmap v1042: 105 pgs: 105 active+clean; 811 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 212 KiB/s rd, 47 MiB/s wr, 331 op/s 2024-09-13T00:52:16.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:16 smithi088 bash[68960]: cluster 2024-09-13T00:52:14.853591+0000 mgr.y (mgr.25304) 491 : cluster [DBG] pgmap v1042: 105 pgs: 105 active+clean; 811 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 212 KiB/s rd, 47 MiB/s wr, 331 op/s 2024-09-13T00:52:17.480 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:17.480 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:17.517 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:17.517 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:17.517 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:17.533 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:17.533 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:17.534 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:17.550 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:17.550 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:17.550 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:17.566 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:17.566 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:17.567 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:17.582 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:17.582 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:17.582 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:17.596 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:17.596 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:17.596 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:17.609 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:17.609 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:17.609 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:17.621 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:17.621 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:17.621 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:17.633 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~54525952,297795584~100663296] 2024-09-13T00:52:17.633 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:17.633 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:17.642 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:17.642 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:17.642 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:17.652 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:17.652 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:17.652 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:17.660 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~125829120,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~96468992] 2024-09-13T00:52:17.661 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to HEAD diff [4194304~62914560,71303168~4194304,83886080~16777216,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~16777216,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:17.661 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~4194304,83886080~16777216,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~16777216,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:17.668 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~71303168,83886080~46137344,138412032~4194304,163577856~25165824,192937984~12582912,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~12582912,314572800~79691776] 2024-09-13T00:52:17.668 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to HEAD diff [4194304~58720256,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:17.668 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~58720256,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:17.674 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~62914560,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~12582912,314572800~16777216,335544320~37748736,377487360~16777216] 2024-09-13T00:52:17.674 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to HEAD diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:17.674 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:17.680 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~20971520,33554432~29360128,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,222298112~12582912,239075328~20971520,264241152~4194304,276824064~8388608,297795584~12582912,314572800~16777216,335544320~37748736,377487360~16777216] 2024-09-13T00:52:17.680 INFO:tasks.workunit.client.0.smithi077.stdout:from 15 to HEAD diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,356515840~12582912,385875968~4194304] 2024-09-13T00:52:17.680 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,356515840~12582912,385875968~4194304] 2024-09-13T00:52:17.684 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~20971520,33554432~29360128,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,201326592~4194304,222298112~12582912,239075328~20971520,264241152~4194304,276824064~8388608,297795584~12582912,314572800~16777216,335544320~16777216,356515840~16777216,385875968~8388608] 2024-09-13T00:52:17.684 INFO:tasks.workunit.client.0.smithi077.stdout:from 16 to HEAD diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,222298112~8388608,239075328~8388608,264241152~4194304,276824064~8388608,297795584~8388608,318767104~8388608,339738624~8388608,356515840~12582912,385875968~4194304] 2024-09-13T00:52:17.684 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,222298112~8388608,239075328~8388608,264241152~4194304,276824064~8388608,297795584~8388608,318767104~8388608,339738624~8388608,356515840~12582912,385875968~4194304] 2024-09-13T00:52:17.687 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~20971520,33554432~29360128,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,222298112~12582912,239075328~12582912,264241152~4194304,276824064~8388608,297795584~12582912,318767104~12582912,339738624~12582912,356515840~16777216,385875968~8388608] 2024-09-13T00:52:17.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:17 smithi077 bash[83127]: cluster 2024-09-13T00:52:16.448728+0000 mon.a (mon.0) 1479 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2024-09-13T00:52:17.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:17 smithi077 bash[80681]: cluster 2024-09-13T00:52:16.448728+0000 mon.a (mon.0) 1479 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2024-09-13T00:52:17.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:17 smithi088 bash[68960]: cluster 2024-09-13T00:52:16.448728+0000 mon.a (mon.0) 1479 : cluster [DBG] osdmap e932: 8 total, 8 up, 8 in 2024-09-13T00:52:18.503 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:18.503 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:18.538 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:18.538 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:18.538 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:18.551 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:18.551 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:18.551 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:18.565 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:18.565 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:18.565 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:18.577 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:18.577 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:18.577 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:18.588 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:18.588 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:18.588 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:18.600 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:18.601 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:18.601 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:18.611 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:18.611 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:18.611 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:18.621 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:18.621 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:18.621 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:18.631 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~54525952,297795584~100663296] 2024-09-13T00:52:18.631 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:18.631 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:18.640 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:18.641 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:18.641 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:18.649 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:18.649 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:18.649 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:18.657 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~125829120,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~96468992] 2024-09-13T00:52:18.657 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to HEAD diff [4194304~62914560,71303168~4194304,83886080~16777216,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~16777216,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:18.658 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~4194304,83886080~16777216,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~16777216,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:18.664 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~71303168,83886080~46137344,138412032~4194304,163577856~25165824,192937984~12582912,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~12582912,314572800~79691776] 2024-09-13T00:52:18.664 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to HEAD diff [4194304~58720256,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:18.664 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~58720256,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:18.670 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~62914560,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~12582912,314572800~16777216,335544320~37748736,377487360~16777216] 2024-09-13T00:52:18.670 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to HEAD diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:18.670 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:18.674 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~20971520,33554432~29360128,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,222298112~12582912,239075328~20971520,264241152~4194304,276824064~8388608,297795584~12582912,314572800~16777216,335544320~37748736,377487360~16777216] 2024-09-13T00:52:18.675 INFO:tasks.workunit.client.0.smithi077.stdout:from 15 to HEAD diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,356515840~12582912,385875968~4194304] 2024-09-13T00:52:18.675 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,356515840~12582912,385875968~4194304] 2024-09-13T00:52:18.677 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~20971520,33554432~29360128,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,184549376~4194304,201326592~4194304,222298112~12582912,239075328~20971520,264241152~4194304,276824064~8388608,297795584~12582912,314572800~16777216,335544320~16777216,356515840~16777216,385875968~8388608] 2024-09-13T00:52:18.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:52:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:52:18] "GET /metrics HTTP/1.1" 200 36694 "" "Prometheus/2.43.0" 2024-09-13T00:52:18.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:18 smithi077 bash[80681]: cluster 2024-09-13T00:52:16.854381+0000 mgr.y (mgr.25304) 492 : cluster [DBG] pgmap v1044: 105 pgs: 105 active+clean; 855 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 353 KiB/s rd, 38 MiB/s wr, 620 op/s 2024-09-13T00:52:18.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:18 smithi077 bash[80681]: cluster 2024-09-13T00:52:17.461800+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2024-09-13T00:52:18.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:18 smithi077 bash[83127]: cluster 2024-09-13T00:52:16.854381+0000 mgr.y (mgr.25304) 492 : cluster [DBG] pgmap v1044: 105 pgs: 105 active+clean; 855 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 353 KiB/s rd, 38 MiB/s wr, 620 op/s 2024-09-13T00:52:18.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:18 smithi077 bash[83127]: cluster 2024-09-13T00:52:17.461800+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2024-09-13T00:52:18.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:18 smithi088 bash[68960]: cluster 2024-09-13T00:52:16.854381+0000 mgr.y (mgr.25304) 492 : cluster [DBG] pgmap v1044: 105 pgs: 105 active+clean; 855 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 353 KiB/s rd, 38 MiB/s wr, 620 op/s 2024-09-13T00:52:18.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:18 smithi088 bash[68960]: cluster 2024-09-13T00:52:17.461800+0000 mon.a (mon.0) 1480 : cluster [DBG] osdmap e933: 8 total, 8 up, 8 in 2024-09-13T00:52:19.509 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:19.509 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:19.537 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:19.538 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:19.538 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:19.551 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:19.551 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:19.552 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:19.563 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:19.564 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:19.564 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:19.574 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:19.575 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:19.575 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:19.586 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:19.586 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:19.586 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:19.598 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:19.598 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:19.598 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:19.610 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:19.610 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:19.610 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:19.621 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:19.621 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:19.622 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:19.632 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~54525952,297795584~100663296] 2024-09-13T00:52:19.632 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:19.632 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:19.642 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:19.642 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:19.643 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:19.651 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:19.651 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:19.651 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:19.659 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~125829120,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~96468992] 2024-09-13T00:52:19.659 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to HEAD diff [4194304~62914560,71303168~4194304,83886080~16777216,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~16777216,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:19.659 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~4194304,83886080~16777216,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~16777216,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:19.666 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~71303168,83886080~46137344,138412032~4194304,163577856~25165824,192937984~12582912,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~12582912,314572800~79691776] 2024-09-13T00:52:19.666 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to HEAD diff [4194304~58720256,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:19.666 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~58720256,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:19.671 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~62914560,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~12582912,314572800~16777216,335544320~37748736,377487360~16777216] 2024-09-13T00:52:19.671 INFO:tasks.workunit.client.0.smithi077.stdout:from 14 to HEAD diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:19.671 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~16777216,33554432~25165824,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,276824064~8388608,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:19.675 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~20971520,33554432~29360128,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,167772160~8388608,180355072~8388608,201326592~4194304,222298112~12582912,239075328~20971520,264241152~4194304,276824064~8388608,297795584~12582912,314572800~16777216,335544320~37748736,377487360~16777216] 2024-09-13T00:52:19.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:19 smithi077 bash[80681]: cluster 2024-09-13T00:52:18.467982+0000 mon.a (mon.0) 1481 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2024-09-13T00:52:19.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:19 smithi077 bash[83127]: cluster 2024-09-13T00:52:18.467982+0000 mon.a (mon.0) 1481 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2024-09-13T00:52:19.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:19 smithi088 bash[68960]: cluster 2024-09-13T00:52:18.467982+0000 mon.a (mon.0) 1481 : cluster [DBG] osdmap e934: 8 total, 8 up, 8 in 2024-09-13T00:52:20.505 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:20.505 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:20.548 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:20.549 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:20.549 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:20.561 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:20.561 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:20.562 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:20.570 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:20.570 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:20.570 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:20.580 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:20.581 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:20.581 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:20.591 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:20.591 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:20.591 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:20.599 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:20.599 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:20.599 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:20.608 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:20.608 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:20.609 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:20.618 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:20.618 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:20.618 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:20.627 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~54525952,297795584~100663296] 2024-09-13T00:52:20.627 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:20.627 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:20.635 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:20.635 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:20.635 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:20.642 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:20.642 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:20.642 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:20.648 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~125829120,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~96468992] 2024-09-13T00:52:20.648 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to HEAD diff [4194304~62914560,71303168~4194304,83886080~16777216,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~16777216,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:20.648 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~4194304,83886080~16777216,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~16777216,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:20.653 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~71303168,83886080~46137344,138412032~4194304,163577856~25165824,192937984~12582912,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~12582912,314572800~79691776] 2024-09-13T00:52:20.653 INFO:tasks.workunit.client.0.smithi077.stdout:from 13 to HEAD diff [4194304~58720256,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:20.653 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~58720256,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~12582912,335544320~12582912,352321536~16777216,377487360~16777216] 2024-09-13T00:52:20.657 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~62914560,71303168~4194304,83886080~4194304,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~12582912,314572800~16777216,335544320~37748736,377487360~16777216] 2024-09-13T00:52:20.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:20 smithi077 bash[83127]: cluster 2024-09-13T00:52:18.855214+0000 mgr.y (mgr.25304) 493 : cluster [DBG] pgmap v1047: 105 pgs: 105 active+clean; 861 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 518 KiB/s rd, 22 MiB/s wr, 941 op/s 2024-09-13T00:52:20.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:20 smithi077 bash[83127]: cluster 2024-09-13T00:52:19.479406+0000 mon.a (mon.0) 1482 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2024-09-13T00:52:20.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:20 smithi077 bash[80681]: cluster 2024-09-13T00:52:18.855214+0000 mgr.y (mgr.25304) 493 : cluster [DBG] pgmap v1047: 105 pgs: 105 active+clean; 861 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 518 KiB/s rd, 22 MiB/s wr, 941 op/s 2024-09-13T00:52:20.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:20 smithi077 bash[80681]: cluster 2024-09-13T00:52:19.479406+0000 mon.a (mon.0) 1482 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2024-09-13T00:52:20.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:20 smithi088 bash[68960]: cluster 2024-09-13T00:52:18.855214+0000 mgr.y (mgr.25304) 493 : cluster [DBG] pgmap v1047: 105 pgs: 105 active+clean; 861 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 518 KiB/s rd, 22 MiB/s wr, 941 op/s 2024-09-13T00:52:20.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:20 smithi088 bash[68960]: cluster 2024-09-13T00:52:19.479406+0000 mon.a (mon.0) 1482 : cluster [DBG] osdmap e935: 8 total, 8 up, 8 in 2024-09-13T00:52:21.536 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:21.537 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:21.571 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:21.571 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:21.571 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:21.581 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:21.581 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:21.581 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:21.591 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:21.591 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:21.591 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:21.600 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:21.600 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:21.601 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:21.609 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:21.609 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:21.609 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:21.617 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:21.617 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:21.617 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:21.624 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:21.624 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:21.624 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:21.631 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:21.631 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:21.631 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:21.636 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~54525952,297795584~100663296] 2024-09-13T00:52:21.637 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:21.637 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:21.642 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:21.642 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:21.642 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:21.646 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:21.646 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:21.647 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:21.651 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~125829120,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~96468992] 2024-09-13T00:52:21.652 INFO:tasks.workunit.client.0.smithi077.stdout:from 12 to HEAD diff [4194304~62914560,71303168~4194304,83886080~16777216,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~16777216,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:21.652 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~4194304,83886080~16777216,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~8388608,314572800~16777216,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:21.654 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~71303168,83886080~46137344,138412032~4194304,163577856~25165824,192937984~12582912,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~12582912,314572800~79691776] 2024-09-13T00:52:21.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:21 smithi077 bash[80681]: cluster 2024-09-13T00:52:20.480336+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2024-09-13T00:52:21.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:21 smithi077 bash[83127]: cluster 2024-09-13T00:52:20.480336+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2024-09-13T00:52:21.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:21 smithi088 bash[68960]: cluster 2024-09-13T00:52:20.480336+0000 mon.a (mon.0) 1483 : cluster [DBG] osdmap e936: 8 total, 8 up, 8 in 2024-09-13T00:52:22.544 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:22.544 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:22.577 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:22.578 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:22.578 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:22.587 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:22.587 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:22.587 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:22.595 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:22.595 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:22.595 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:22.603 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:22.603 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:22.603 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:22.610 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:22.610 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:22.610 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:22.616 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:22.617 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:22.617 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:22.623 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:22.623 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:22.623 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:22.629 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:22.629 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:22.629 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:22.633 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~54525952,297795584~100663296] 2024-09-13T00:52:22.634 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:22.634 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:22.638 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:22.638 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:22.638 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:22.642 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:22.642 INFO:tasks.workunit.client.0.smithi077.stdout:from 11 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:22.642 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~25165824,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~12582912,352321536~20971520,377487360~16777216] 2024-09-13T00:52:22.644 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~125829120,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~96468992] 2024-09-13T00:52:22.696 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:52:22 smithi088 bash[61791]: ts=2024-09-13T00:52:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:52:22.697 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:22 smithi088 bash[68960]: cluster 2024-09-13T00:52:20.855894+0000 mgr.y (mgr.25304) 494 : cluster [DBG] pgmap v1050: 105 pgs: 105 active+clean; 848 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 436 KiB/s rd, 1.9 MiB/s wr, 803 op/s 2024-09-13T00:52:22.697 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:22 smithi088 bash[68960]: cluster 2024-09-13T00:52:21.502446+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2024-09-13T00:52:22.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:22 smithi077 bash[80681]: cluster 2024-09-13T00:52:20.855894+0000 mgr.y (mgr.25304) 494 : cluster [DBG] pgmap v1050: 105 pgs: 105 active+clean; 848 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 436 KiB/s rd, 1.9 MiB/s wr, 803 op/s 2024-09-13T00:52:22.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:22 smithi077 bash[80681]: cluster 2024-09-13T00:52:21.502446+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2024-09-13T00:52:22.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:22 smithi077 bash[83127]: cluster 2024-09-13T00:52:20.855894+0000 mgr.y (mgr.25304) 494 : cluster [DBG] pgmap v1050: 105 pgs: 105 active+clean; 848 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 436 KiB/s rd, 1.9 MiB/s wr, 803 op/s 2024-09-13T00:52:22.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:22 smithi077 bash[83127]: cluster 2024-09-13T00:52:21.502446+0000 mon.a (mon.0) 1484 : cluster [DBG] osdmap e937: 8 total, 8 up, 8 in 2024-09-13T00:52:23.558 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:23.559 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:23.592 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:23.592 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:23.592 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:23.599 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:23.599 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:23.599 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:23.606 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:23.606 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:23.606 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:23.613 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:23.613 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:23.613 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:23.620 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:23.620 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:23.620 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:23.625 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:23.625 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:23.625 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:23.630 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:23.630 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:23.630 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:23.635 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:23.635 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:23.635 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:23.639 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~54525952,297795584~100663296] 2024-09-13T00:52:23.639 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:23.639 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:23.642 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:23.642 INFO:tasks.workunit.client.0.smithi077.stdout:from 10 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:23.642 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,163577856~25165824,192937984~12582912,222298112~8388608,239075328~8388608,251658240~8388608,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:23.645 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,163577856~41943040,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:23.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:23 smithi077 bash[80681]: cluster 2024-09-13T00:52:22.511669+0000 mon.a (mon.0) 1485 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2024-09-13T00:52:23.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:23 smithi077 bash[83127]: cluster 2024-09-13T00:52:22.511669+0000 mon.a (mon.0) 1485 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2024-09-13T00:52:23.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:23 smithi088 bash[68960]: cluster 2024-09-13T00:52:22.511669+0000 mon.a (mon.0) 1485 : cluster [DBG] osdmap e938: 8 total, 8 up, 8 in 2024-09-13T00:52:24.572 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:24.572 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:24.597 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:24.597 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:24.597 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:24.605 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:24.605 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:24.605 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:24.611 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:24.611 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:24.611 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:24.617 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:24.617 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:24.617 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:24.622 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:24.622 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:24.622 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:24.626 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:24.626 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:24.626 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:24.630 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:24.630 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:24.630 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:24.633 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:24.633 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:24.633 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:24.636 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~54525952,297795584~100663296] 2024-09-13T00:52:24.636 INFO:tasks.workunit.client.0.smithi077.stdout:from 9 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:24.636 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~20971520,264241152~4194304,272629760~12582912,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:24.637 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~20971520,264241152~4194304,272629760~12582912,297795584~100663296] 2024-09-13T00:52:24.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:24 smithi077 bash[80681]: cluster 2024-09-13T00:52:22.856675+0000 mgr.y (mgr.25304) 495 : cluster [DBG] pgmap v1053: 105 pgs: 105 active+clean; 835 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 335 KiB/s rd, 4.7 KiB/s wr, 599 op/s 2024-09-13T00:52:24.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:24 smithi077 bash[80681]: cluster 2024-09-13T00:52:23.523060+0000 mon.a (mon.0) 1486 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2024-09-13T00:52:24.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:24 smithi077 bash[83127]: cluster 2024-09-13T00:52:22.856675+0000 mgr.y (mgr.25304) 495 : cluster [DBG] pgmap v1053: 105 pgs: 105 active+clean; 835 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 335 KiB/s rd, 4.7 KiB/s wr, 599 op/s 2024-09-13T00:52:24.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:24 smithi077 bash[83127]: cluster 2024-09-13T00:52:23.523060+0000 mon.a (mon.0) 1486 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2024-09-13T00:52:24.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:24 smithi088 bash[68960]: cluster 2024-09-13T00:52:22.856675+0000 mgr.y (mgr.25304) 495 : cluster [DBG] pgmap v1053: 105 pgs: 105 active+clean; 835 MiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 335 KiB/s rd, 4.7 KiB/s wr, 599 op/s 2024-09-13T00:52:24.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:24 smithi088 bash[68960]: cluster 2024-09-13T00:52:23.523060+0000 mon.a (mon.0) 1486 : cluster [DBG] osdmap e939: 8 total, 8 up, 8 in 2024-09-13T00:52:25.575 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:25.576 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:25.609 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:25.609 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:25.609 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:25.616 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:25.616 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:25.616 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:25.621 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:25.621 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:25.621 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:25.626 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:25.627 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:25.627 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:25.630 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:25.630 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:25.630 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:25.634 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:25.634 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:25.634 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:25.637 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:25.637 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:25.638 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:25.640 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:25.640 INFO:tasks.workunit.client.0.smithi077.stdout:from 8 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:25.640 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~41943040,285212672~8388608,297795584~33554432,335544320~37748736,377487360~20971520] 2024-09-13T00:52:25.642 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~54525952,297795584~100663296] 2024-09-13T00:52:25.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:25 smithi088 bash[68960]: cluster 2024-09-13T00:52:24.523516+0000 mon.a (mon.0) 1487 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2024-09-13T00:52:25.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:25 smithi088 bash[68960]: cluster 2024-09-13T00:52:24.857462+0000 mgr.y (mgr.25304) 496 : cluster [DBG] pgmap v1056: 105 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 102 active+clean; 791 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 298 KiB/s rd, 6.2 KiB/s wr, 519 op/s 2024-09-13T00:52:26.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:25 smithi077 bash[80681]: cluster 2024-09-13T00:52:24.523516+0000 mon.a (mon.0) 1487 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2024-09-13T00:52:26.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:25 smithi077 bash[80681]: cluster 2024-09-13T00:52:24.857462+0000 mgr.y (mgr.25304) 496 : cluster [DBG] pgmap v1056: 105 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 102 active+clean; 791 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 298 KiB/s rd, 6.2 KiB/s wr, 519 op/s 2024-09-13T00:52:26.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:25 smithi077 bash[83127]: cluster 2024-09-13T00:52:24.523516+0000 mon.a (mon.0) 1487 : cluster [DBG] osdmap e940: 8 total, 8 up, 8 in 2024-09-13T00:52:26.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:25 smithi077 bash[83127]: cluster 2024-09-13T00:52:24.857462+0000 mgr.y (mgr.25304) 496 : cluster [DBG] pgmap v1056: 105 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 102 active+clean; 791 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 298 KiB/s rd, 6.2 KiB/s wr, 519 op/s 2024-09-13T00:52:26.171 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:26.171 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:26.192 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:26.192 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:26.192 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:26.200 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:26.200 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:26.200 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:26.205 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:26.206 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:26.206 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:26.210 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:26.210 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:26.210 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:26.214 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:26.214 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:26.214 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:26.217 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:26.217 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:26.217 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:26.219 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:26.219 INFO:tasks.workunit.client.0.smithi077.stdout:from 7 to HEAD diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:26.219 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [4194304~62914560,71303168~16777216,92274688~8388608,104857600~29360128,138412032~4194304,155189248~33554432,192937984~16777216,213909504~4194304,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:26.221 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [4194304~130023424,138412032~4194304,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:26.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:26 smithi088 bash[68960]: cluster 2024-09-13T00:52:25.545847+0000 mon.a (mon.0) 1488 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2024-09-13T00:52:26.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:26 smithi088 bash[68960]: cluster 2024-09-13T00:52:26.126798+0000 mon.a (mon.0) 1489 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2024-09-13T00:52:27.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:26 smithi077 bash[80681]: cluster 2024-09-13T00:52:25.545847+0000 mon.a (mon.0) 1488 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2024-09-13T00:52:27.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:26 smithi077 bash[80681]: cluster 2024-09-13T00:52:26.126798+0000 mon.a (mon.0) 1489 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2024-09-13T00:52:27.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:26 smithi077 bash[83127]: cluster 2024-09-13T00:52:25.545847+0000 mon.a (mon.0) 1488 : cluster [DBG] osdmap e941: 8 total, 8 up, 8 in 2024-09-13T00:52:27.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:26 smithi077 bash[83127]: cluster 2024-09-13T00:52:26.126798+0000 mon.a (mon.0) 1489 : cluster [DBG] osdmap e942: 8 total, 8 up, 8 in 2024-09-13T00:52:27.168 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:27.168 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:27.197 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:27.197 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:27.197 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:27.202 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:27.202 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:27.202 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:27.206 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:27.206 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:27.207 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:27.210 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:27.210 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:27.210 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:27.213 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:27.213 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:27.213 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:27.216 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:27.216 INFO:tasks.workunit.client.0.smithi077.stdout:from 6 to HEAD diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:27.216 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~67108864,71303168~29360128,104857600~37748736,155189248~33554432,192937984~25165824,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:27.218 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:28.182 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:28.182 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:28.205 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:28.206 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:28.206 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:28.211 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:28.211 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:28.211 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:28.215 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:28.216 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:28.216 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:28.219 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:28.219 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:28.219 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:28.221 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:28.222 INFO:tasks.workunit.client.0.smithi077.stdout:from 5 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:28.222 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,155189248~62914560,222298112~8388608,239075328~134217728,377487360~29360128] 2024-09-13T00:52:28.223 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,155189248~62914560,222298112~12582912,239075328~171966464] 2024-09-13T00:52:28.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:28 smithi077 bash[80681]: cluster 2024-09-13T00:52:26.858319+0000 mgr.y (mgr.25304) 497 : cluster [DBG] pgmap v1059: 105 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 102 active+clean; 733 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 276 KiB/s rd, 6.2 KiB/s wr, 476 op/s 2024-09-13T00:52:28.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:28 smithi077 bash[80681]: cluster 2024-09-13T00:52:27.129620+0000 mon.a (mon.0) 1490 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2024-09-13T00:52:28.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:28 smithi077 bash[83127]: cluster 2024-09-13T00:52:26.858319+0000 mgr.y (mgr.25304) 497 : cluster [DBG] pgmap v1059: 105 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 102 active+clean; 733 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 276 KiB/s rd, 6.2 KiB/s wr, 476 op/s 2024-09-13T00:52:28.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:28 smithi077 bash[83127]: cluster 2024-09-13T00:52:27.129620+0000 mon.a (mon.0) 1490 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2024-09-13T00:52:28.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:28 smithi088 bash[68960]: cluster 2024-09-13T00:52:26.858319+0000 mgr.y (mgr.25304) 497 : cluster [DBG] pgmap v1059: 105 pgs: 2 active+clean+snaptrim, 1 active+clean+snaptrim_wait, 102 active+clean; 733 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 276 KiB/s rd, 6.2 KiB/s wr, 476 op/s 2024-09-13T00:52:28.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:28 smithi088 bash[68960]: cluster 2024-09-13T00:52:27.129620+0000 mon.a (mon.0) 1490 : cluster [DBG] osdmap e943: 8 total, 8 up, 8 in 2024-09-13T00:52:28.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:52:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:52:28] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.43.0" 2024-09-13T00:52:29.190 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:29.190 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:29.215 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:29.215 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:29.215 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:29.220 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:29.220 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:29.221 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:29.224 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:29.224 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:29.224 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:29.227 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:29.227 INFO:tasks.workunit.client.0.smithi077.stdout:from 4 to HEAD diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:29.227 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~8388608,104857600~37748736,146800640~71303168,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:29.229 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:29.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:29 smithi077 bash[80681]: cluster 2024-09-13T00:52:28.134251+0000 mon.a (mon.0) 1491 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2024-09-13T00:52:29.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:29 smithi077 bash[80681]: audit 2024-09-13T00:52:28.428428+0000 mon.a (mon.0) 1492 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:52:29.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:29 smithi077 bash[83127]: cluster 2024-09-13T00:52:28.134251+0000 mon.a (mon.0) 1491 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2024-09-13T00:52:29.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:29 smithi077 bash[83127]: audit 2024-09-13T00:52:28.428428+0000 mon.a (mon.0) 1492 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:52:29.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:29 smithi088 bash[68960]: cluster 2024-09-13T00:52:28.134251+0000 mon.a (mon.0) 1491 : cluster [DBG] osdmap e944: 8 total, 8 up, 8 in 2024-09-13T00:52:29.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:29 smithi088 bash[68960]: audit 2024-09-13T00:52:28.428428+0000 mon.a (mon.0) 1492 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:52:30.193 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:30.193 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:30.222 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:30.222 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:30.222 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:30.227 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:30.227 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:30.227 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:30.231 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:30.231 INFO:tasks.workunit.client.0.smithi077.stdout:from 3 to HEAD diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:30.231 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~88080384,92274688~16777216,113246208~29360128,146800640~41943040,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:30.234 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~142606336,146800640~71303168,222298112~12582912,239075328~171966464] 2024-09-13T00:52:30.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:30 smithi077 bash[80681]: cluster 2024-09-13T00:52:28.859131+0000 mgr.y (mgr.25304) 498 : cluster [DBG] pgmap v1062: 105 pgs: 105 active+clean; 687 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 167 op/s 2024-09-13T00:52:30.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:30 smithi077 bash[80681]: cluster 2024-09-13T00:52:29.164227+0000 mon.a (mon.0) 1493 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2024-09-13T00:52:30.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:30 smithi077 bash[83127]: cluster 2024-09-13T00:52:28.859131+0000 mgr.y (mgr.25304) 498 : cluster [DBG] pgmap v1062: 105 pgs: 105 active+clean; 687 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 167 op/s 2024-09-13T00:52:30.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:30 smithi077 bash[83127]: cluster 2024-09-13T00:52:29.164227+0000 mon.a (mon.0) 1493 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2024-09-13T00:52:30.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:30 smithi088 bash[68960]: cluster 2024-09-13T00:52:28.859131+0000 mgr.y (mgr.25304) 498 : cluster [DBG] pgmap v1062: 105 pgs: 105 active+clean; 687 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 84 KiB/s rd, 0 B/s wr, 167 op/s 2024-09-13T00:52:30.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:30 smithi088 bash[68960]: cluster 2024-09-13T00:52:29.164227+0000 mon.a (mon.0) 1493 : cluster [DBG] osdmap e945: 8 total, 8 up, 8 in 2024-09-13T00:52:31.157 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:31.157 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:31.190 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:31.190 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:31.190 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:31.193 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:31.193 INFO:tasks.workunit.client.0.smithi077.stdout:from 2 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:31.193 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~134217728,377487360~33554432] 2024-09-13T00:52:31.195 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:31.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:31 smithi077 bash[80681]: cluster 2024-09-13T00:52:30.163723+0000 mon.a (mon.0) 1494 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2024-09-13T00:52:31.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:31 smithi077 bash[80681]: cluster 2024-09-13T00:52:31.130466+0000 mon.a (mon.0) 1495 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2024-09-13T00:52:31.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:31 smithi077 bash[83127]: cluster 2024-09-13T00:52:30.163723+0000 mon.a (mon.0) 1494 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2024-09-13T00:52:31.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:31 smithi077 bash[83127]: cluster 2024-09-13T00:52:31.130466+0000 mon.a (mon.0) 1495 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2024-09-13T00:52:31.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:31 smithi088 bash[68960]: cluster 2024-09-13T00:52:30.163723+0000 mon.a (mon.0) 1494 : cluster [DBG] osdmap e946: 8 total, 8 up, 8 in 2024-09-13T00:52:31.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:31 smithi088 bash[68960]: cluster 2024-09-13T00:52:31.130466+0000 mon.a (mon.0) 1495 : cluster [DBG] osdmap e947: 8 total, 8 up, 8 in 2024-09-13T00:52:32.217 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:32.217 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:32.245 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:32.245 INFO:tasks.workunit.client.0.smithi077.stdout:from 1 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:32.245 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:32.247 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:32.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:32 smithi077 bash[80681]: cluster 2024-09-13T00:52:30.859903+0000 mgr.y (mgr.25304) 499 : cluster [DBG] pgmap v1065: 105 pgs: 105 active+clean; 639 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 147 KiB/s rd, 7.2 KiB/s wr, 231 op/s 2024-09-13T00:52:32.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:32 smithi077 bash[83127]: cluster 2024-09-13T00:52:30.859903+0000 mgr.y (mgr.25304) 499 : cluster [DBG] pgmap v1065: 105 pgs: 105 active+clean; 639 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 147 KiB/s rd, 7.2 KiB/s wr, 231 op/s 2024-09-13T00:52:32.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:32 smithi088 bash[68960]: cluster 2024-09-13T00:52:30.859903+0000 mgr.y (mgr.25304) 499 : cluster [DBG] pgmap v1065: 105 pgs: 105 active+clean; 639 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 147 KiB/s rd, 7.2 KiB/s wr, 231 op/s 2024-09-13T00:52:32.602 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:52:32 smithi088 bash[60519]: logger=sqlstore.transactions t=2024-09-13T00:52:32.366954917Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-13T00:52:32.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:52:32 smithi088 bash[61791]: ts=2024-09-13T00:52:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:52:33.252 INFO:tasks.workunit.client.0.smithi077.stdout:from 0 to HEAD diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:33.252 INFO:tasks.workunit.client.0.smithi077.stdout: limited diff [0~109051904,113246208~75497472,192937984~25165824,222298112~8388608,239075328~171966464] 2024-09-13T00:52:33.268 INFO:tasks.workunit.client.0.smithi077.stdout: actual was [0~218103808,222298112~12582912,239075328~171966464] 2024-09-13T00:52:33.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:33 smithi077 bash[80681]: cluster 2024-09-13T00:52:32.192528+0000 mon.a (mon.0) 1496 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2024-09-13T00:52:33.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:33 smithi077 bash[83127]: cluster 2024-09-13T00:52:32.192528+0000 mon.a (mon.0) 1496 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2024-09-13T00:52:33.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:33 smithi088 bash[68960]: cluster 2024-09-13T00:52:32.192528+0000 mon.a (mon.0) 1496 : cluster [DBG] osdmap e948: 8 total, 8 up, 8 in 2024-09-13T00:52:34.269 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/1.DiffIterateStress (40025 ms) 2024-09-13T00:52:34.269 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/1.DiffIterateRegression6926 2024-09-13T00:52:34.269 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:52:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:34 smithi077 bash[80681]: cluster 2024-09-13T00:52:32.860645+0000 mgr.y (mgr.25304) 500 : cluster [DBG] pgmap v1068: 105 pgs: 105 active+clean; 602 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 132 KiB/s rd, 7.2 KiB/s wr, 200 op/s 2024-09-13T00:52:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:34 smithi077 bash[80681]: cluster 2024-09-13T00:52:33.212539+0000 mon.a (mon.0) 1497 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2024-09-13T00:52:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:34 smithi077 bash[83127]: cluster 2024-09-13T00:52:32.860645+0000 mgr.y (mgr.25304) 500 : cluster [DBG] pgmap v1068: 105 pgs: 105 active+clean; 602 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 132 KiB/s rd, 7.2 KiB/s wr, 200 op/s 2024-09-13T00:52:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:34 smithi077 bash[83127]: cluster 2024-09-13T00:52:33.212539+0000 mon.a (mon.0) 1497 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2024-09-13T00:52:34.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:34 smithi088 bash[68960]: cluster 2024-09-13T00:52:32.860645+0000 mgr.y (mgr.25304) 500 : cluster [DBG] pgmap v1068: 105 pgs: 105 active+clean; 602 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 132 KiB/s rd, 7.2 KiB/s wr, 200 op/s 2024-09-13T00:52:34.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:34 smithi088 bash[68960]: cluster 2024-09-13T00:52:33.212539+0000 mon.a (mon.0) 1497 : cluster [DBG] osdmap e949: 8 total, 8 up, 8 in 2024-09-13T00:52:35.267 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 0~4194304 2024-09-13T00:52:35.278 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/1.DiffIterateRegression6926 (1009 ms) 2024-09-13T00:52:35.278 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParent 2024-09-13T00:52:35.278 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:52:35.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:35 smithi077 bash[80681]: cluster 2024-09-13T00:52:34.219718+0000 mon.a (mon.0) 1498 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2024-09-13T00:52:35.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:35 smithi077 bash[83127]: cluster 2024-09-13T00:52:34.219718+0000 mon.a (mon.0) 1498 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2024-09-13T00:52:35.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:35 smithi088 bash[68960]: cluster 2024-09-13T00:52:34.219718+0000 mon.a (mon.0) 1498 : cluster [DBG] osdmap e950: 8 total, 8 up, 8 in 2024-09-13T00:52:36.300 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 0~4194304 2024-09-13T00:52:36.301 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 4194304~4194304 2024-09-13T00:52:36.301 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 8388608~4194304 2024-09-13T00:52:36.301 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 12582912~4194304 2024-09-13T00:52:36.301 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 16777216~4194304 2024-09-13T00:52:36.381 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 0~4194304 2024-09-13T00:52:36.381 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 4194304~4194304 2024-09-13T00:52:36.381 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 8388608~4194304 2024-09-13T00:52:36.381 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 16777216~4194304 2024-09-13T00:52:36.411 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/1.DiffIterateParent (1134 ms) 2024-09-13T00:52:36.411 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/1.DiffIterateIgnoreParent 2024-09-13T00:52:36.411 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:52:36.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:36 smithi077 bash[80681]: cluster 2024-09-13T00:52:34.861676+0000 mgr.y (mgr.25304) 501 : cluster [DBG] pgmap v1071: 105 pgs: 105 active+clean; 496 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 5.0 KiB/s wr, 121 op/s 2024-09-13T00:52:36.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:36 smithi077 bash[80681]: cluster 2024-09-13T00:52:35.231725+0000 mon.a (mon.0) 1499 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2024-09-13T00:52:36.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:36 smithi077 bash[80681]: cluster 2024-09-13T00:52:36.132118+0000 mon.a (mon.0) 1500 : cluster [DBG] osdmap e952: 8 total, 8 up, 8 in 2024-09-13T00:52:36.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:36 smithi077 bash[83127]: cluster 2024-09-13T00:52:34.861676+0000 mgr.y (mgr.25304) 501 : cluster [DBG] pgmap v1071: 105 pgs: 105 active+clean; 496 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 5.0 KiB/s wr, 121 op/s 2024-09-13T00:52:36.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:36 smithi077 bash[83127]: cluster 2024-09-13T00:52:35.231725+0000 mon.a (mon.0) 1499 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2024-09-13T00:52:36.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:36 smithi077 bash[83127]: cluster 2024-09-13T00:52:36.132118+0000 mon.a (mon.0) 1500 : cluster [DBG] osdmap e952: 8 total, 8 up, 8 in 2024-09-13T00:52:36.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:36 smithi088 bash[68960]: cluster 2024-09-13T00:52:34.861676+0000 mgr.y (mgr.25304) 501 : cluster [DBG] pgmap v1071: 105 pgs: 105 active+clean; 496 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 5.0 KiB/s wr, 121 op/s 2024-09-13T00:52:36.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:36 smithi088 bash[68960]: cluster 2024-09-13T00:52:35.231725+0000 mon.a (mon.0) 1499 : cluster [DBG] osdmap e951: 8 total, 8 up, 8 in 2024-09-13T00:52:36.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:36 smithi088 bash[68960]: cluster 2024-09-13T00:52:36.132118+0000 mon.a (mon.0) 1500 : cluster [DBG] osdmap e952: 8 total, 8 up, 8 in 2024-09-13T00:52:37.506 INFO:tasks.workunit.client.0.smithi077.stdout: wrote [0~16777216] to clone 2024-09-13T00:52:37.507 INFO:tasks.workunit.client.0.smithi077.stdout: diff was [0~16777216] 2024-09-13T00:52:37.517 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/1.DiffIterateIgnoreParent (1106 ms) 2024-09-13T00:52:37.517 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/1.DiffIterateCallbackError 2024-09-13T00:52:37.517 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:52:37.566 INFO:tasks.workunit.client.0.smithi077.stdout: wrote [1595655~55621,2740332~34365,4484089~23963,6021549~66251,9279746~77558,10213185~34040,10514523~50763,14698236~48214,15686262~32637,19511966~93239] 2024-09-13T00:52:37.592 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/1.DiffIterateCallbackError (75 ms) 2024-09-13T00:52:37.593 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParentDiscard 2024-09-13T00:52:37.593 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:52:38.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:38 smithi077 bash[83127]: cluster 2024-09-13T00:52:36.862553+0000 mgr.y (mgr.25304) 502 : cluster [DBG] pgmap v1074: 105 pgs: 105 active+clean; 464 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.2 KiB/s wr, 124 op/s 2024-09-13T00:52:38.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:38 smithi077 bash[83127]: cluster 2024-09-13T00:52:37.239494+0000 mon.a (mon.0) 1501 : cluster [DBG] osdmap e953: 8 total, 8 up, 8 in 2024-09-13T00:52:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:38 smithi077 bash[80681]: cluster 2024-09-13T00:52:36.862553+0000 mgr.y (mgr.25304) 502 : cluster [DBG] pgmap v1074: 105 pgs: 105 active+clean; 464 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.2 KiB/s wr, 124 op/s 2024-09-13T00:52:38.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:38 smithi077 bash[80681]: cluster 2024-09-13T00:52:37.239494+0000 mon.a (mon.0) 1501 : cluster [DBG] osdmap e953: 8 total, 8 up, 8 in 2024-09-13T00:52:38.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:38 smithi088 bash[68960]: cluster 2024-09-13T00:52:36.862553+0000 mgr.y (mgr.25304) 502 : cluster [DBG] pgmap v1074: 105 pgs: 105 active+clean; 464 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 80 KiB/s rd, 7.2 KiB/s wr, 124 op/s 2024-09-13T00:52:38.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:38 smithi088 bash[68960]: cluster 2024-09-13T00:52:37.239494+0000 mon.a (mon.0) 1501 : cluster [DBG] osdmap e953: 8 total, 8 up, 8 in 2024-09-13T00:52:38.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:52:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:52:38] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.43.0" 2024-09-13T00:52:39.487 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/1.DiffIterateParentDiscard (1895 ms) 2024-09-13T00:52:39.487 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnalignedSmall 2024-09-13T00:52:39.487 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:52:39.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:39 smithi077 bash[80681]: cluster 2024-09-13T00:52:38.252293+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e954: 8 total, 8 up, 8 in 2024-09-13T00:52:39.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:39 smithi077 bash[83127]: cluster 2024-09-13T00:52:38.252293+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e954: 8 total, 8 up, 8 in 2024-09-13T00:52:39.579 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 5000005~1234 2024-09-13T00:52:39.587 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnalignedSmall (100 ms) 2024-09-13T00:52:39.587 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnaligned 2024-09-13T00:52:39.587 INFO:tasks.workunit.client.0.smithi077.stdout:using new format! 2024-09-13T00:52:39.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:39 smithi088 bash[68960]: cluster 2024-09-13T00:52:38.252293+0000 mon.a (mon.0) 1502 : cluster [DBG] osdmap e954: 8 total, 8 up, 8 in 2024-09-13T00:52:39.725 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 8376263~12345 2024-09-13T00:52:39.725 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 8388608~4194304 2024-09-13T00:52:39.725 INFO:tasks.workunit.client.0.smithi077.stdout:iterate_cb 12582912~54321 2024-09-13T00:52:39.738 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnaligned (150 ms) 2024-09-13T00:52:39.738 INFO:tasks.workunit.client.0.smithi077.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStriping 2024-09-13T00:52:39.738 INFO:tasks.workunit.client.0.smithi077.stdout:SKIPPING 2024-09-13T00:52:39.738 INFO:tasks.workunit.client.0.smithi077.stdout:[ OK ] DiffIterateTest/1.DiffIterateStriping (0 ms) 2024-09-13T00:52:40.266 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] 11 tests from DiffIterateTest/1 (53557 ms total) 2024-09-13T00:52:40.266 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:52:40.266 INFO:tasks.workunit.client.0.smithi077.stdout:[----------] Global test environment tear-down 2024-09-13T00:52:40.267 INFO:tasks.workunit.client.0.smithi077.stdout:[==========] 406 tests from 15 test suites ran. (1074717 ms total) 2024-09-13T00:52:40.267 INFO:tasks.workunit.client.0.smithi077.stdout:[ PASSED ] 406 tests. 2024-09-13T00:52:40.267 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:52:40.267 INFO:tasks.workunit.client.0.smithi077.stdout: YOU HAVE 2 DISABLED TESTS 2024-09-13T00:52:40.267 INFO:tasks.workunit.client.0.smithi077.stdout: 2024-09-13T00:52:40.301 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-13T00:52:40.301 DEBUG:teuthology.orchestra.run.smithi077:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-13T00:52:40.356 INFO:tasks.workunit:Stopping ['rbd/test_librbd.sh'] on client.0... 2024-09-13T00:52:40.356 DEBUG:teuthology.orchestra.run.smithi077:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-13T00:52:40.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:40 smithi077 bash[83127]: cluster 2024-09-13T00:52:38.863343+0000 mgr.y (mgr.25304) 503 : cluster [DBG] pgmap v1077: 105 pgs: 105 active+clean; 472 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.6 MiB/s wr, 60 op/s 2024-09-13T00:52:40.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:40 smithi077 bash[83127]: cluster 2024-09-13T00:52:39.259965+0000 mon.a (mon.0) 1503 : cluster [DBG] osdmap e955: 8 total, 8 up, 8 in 2024-09-13T00:52:40.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:40 smithi077 bash[80681]: cluster 2024-09-13T00:52:38.863343+0000 mgr.y (mgr.25304) 503 : cluster [DBG] pgmap v1077: 105 pgs: 105 active+clean; 472 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.6 MiB/s wr, 60 op/s 2024-09-13T00:52:40.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:40 smithi077 bash[80681]: cluster 2024-09-13T00:52:39.259965+0000 mon.a (mon.0) 1503 : cluster [DBG] osdmap e955: 8 total, 8 up, 8 in 2024-09-13T00:52:40.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:40 smithi088 bash[68960]: cluster 2024-09-13T00:52:38.863343+0000 mgr.y (mgr.25304) 503 : cluster [DBG] pgmap v1077: 105 pgs: 105 active+clean; 472 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.6 MiB/s wr, 60 op/s 2024-09-13T00:52:40.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:40 smithi088 bash[68960]: cluster 2024-09-13T00:52:39.259965+0000 mon.a (mon.0) 1503 : cluster [DBG] osdmap e955: 8 total, 8 up, 8 in 2024-09-13T00:52:40.625 DEBUG:teuthology.parallel:result is None 2024-09-13T00:52:40.625 DEBUG:teuthology.orchestra.run.smithi077:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T00:52:40.638 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T00:52:40.638 DEBUG:teuthology.orchestra.run.smithi077:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-09-13T00:52:40.692 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-09-13T00:52:40.693 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-13T00:52:40.693 INFO:teuthology.task.print:**** done rbd/test_librbd.sh 7-workload 2024-09-13T00:52:40.693 DEBUG:teuthology.parallel:result is None 2024-09-13T00:52:40.693 INFO:teuthology.run_tasks:Running task print... 2024-09-13T00:52:40.706 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-09-13T00:52:40.706 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-13T00:52:40.719 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-13T00:52:40.719 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-13T00:52:40.720 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-13T00:52:41.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:41 smithi077 bash[83127]: cluster 2024-09-13T00:52:40.262102+0000 mon.a (mon.0) 1504 : cluster [DBG] osdmap e956: 8 total, 8 up, 8 in 2024-09-13T00:52:41.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:41 smithi077 bash[80681]: cluster 2024-09-13T00:52:40.262102+0000 mon.a (mon.0) 1504 : cluster [DBG] osdmap e956: 8 total, 8 up, 8 in 2024-09-13T00:52:41.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:41 smithi088 bash[68960]: cluster 2024-09-13T00:52:40.262102+0000 mon.a (mon.0) 1504 : cluster [DBG] osdmap e956: 8 total, 8 up, 8 in 2024-09-13T00:52:42.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:42 smithi077 bash[80681]: cluster 2024-09-13T00:52:40.864083+0000 mgr.y (mgr.25304) 504 : cluster [DBG] pgmap v1080: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:52:42.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:42 smithi077 bash[83127]: cluster 2024-09-13T00:52:40.864083+0000 mgr.y (mgr.25304) 504 : cluster [DBG] pgmap v1080: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:52:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:42 smithi088 bash[68960]: cluster 2024-09-13T00:52:40.864083+0000 mgr.y (mgr.25304) 504 : cluster [DBG] pgmap v1080: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:52:42.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:52:42 smithi088 bash[61791]: ts=2024-09-13T00:52:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:52:44.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:44 smithi088 bash[68960]: cluster 2024-09-13T00:52:42.864851+0000 mgr.y (mgr.25304) 505 : cluster [DBG] pgmap v1081: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:52:44.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:44 smithi088 bash[68960]: audit 2024-09-13T00:52:43.428963+0000 mon.a (mon.0) 1505 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:52:44.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:44 smithi077 bash[80681]: cluster 2024-09-13T00:52:42.864851+0000 mgr.y (mgr.25304) 505 : cluster [DBG] pgmap v1081: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:52:44.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:44 smithi077 bash[80681]: audit 2024-09-13T00:52:43.428963+0000 mon.a (mon.0) 1505 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:52:44.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:44 smithi077 bash[83127]: cluster 2024-09-13T00:52:42.864851+0000 mgr.y (mgr.25304) 505 : cluster [DBG] pgmap v1081: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-13T00:52:44.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:44 smithi077 bash[83127]: audit 2024-09-13T00:52:43.428963+0000 mon.a (mon.0) 1505 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:52:45.519 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:52:46.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:46 smithi088 bash[68960]: cluster 2024-09-13T00:52:44.865858+0000 mgr.y (mgr.25304) 506 : cluster [DBG] pgmap v1082: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:46.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:46 smithi077 bash[83127]: cluster 2024-09-13T00:52:44.865858+0000 mgr.y (mgr.25304) 506 : cluster [DBG] pgmap v1082: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:46.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:46 smithi077 bash[80681]: cluster 2024-09-13T00:52:44.865858+0000 mgr.y (mgr.25304) 506 : cluster [DBG] pgmap v1082: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:48.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:48 smithi088 bash[68960]: cluster 2024-09-13T00:52:46.866640+0000 mgr.y (mgr.25304) 507 : cluster [DBG] pgmap v1083: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:48.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:48 smithi077 bash[80681]: cluster 2024-09-13T00:52:46.866640+0000 mgr.y (mgr.25304) 507 : cluster [DBG] pgmap v1083: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:48.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:52:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:52:48] "GET /metrics HTTP/1.1" 200 34265 "" "Prometheus/2.43.0" 2024-09-13T00:52:48.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:48 smithi077 bash[83127]: cluster 2024-09-13T00:52:46.866640+0000 mgr.y (mgr.25304) 507 : cluster [DBG] pgmap v1083: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:50.117 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:52:50.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:50 smithi077 bash[80681]: cluster 2024-09-13T00:52:48.867422+0000 mgr.y (mgr.25304) 508 : cluster [DBG] pgmap v1084: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:50.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:50 smithi077 bash[80681]: audit 2024-09-13T00:52:50.114890+0000 mon.a (mon.0) 1506 : audit [DBG] from='client.? 172.21.15.77:0/1618810195' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T00:52:50.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:50 smithi077 bash[83127]: cluster 2024-09-13T00:52:48.867422+0000 mgr.y (mgr.25304) 508 : cluster [DBG] pgmap v1084: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:50.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:50 smithi077 bash[83127]: audit 2024-09-13T00:52:50.114890+0000 mon.a (mon.0) 1506 : audit [DBG] from='client.? 172.21.15.77:0/1618810195' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T00:52:50.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:50 smithi088 bash[68960]: cluster 2024-09-13T00:52:48.867422+0000 mgr.y (mgr.25304) 508 : cluster [DBG] pgmap v1084: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:50.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:50 smithi088 bash[68960]: audit 2024-09-13T00:52:50.114890+0000 mon.a (mon.0) 1506 : audit [DBG] from='client.? 172.21.15.77:0/1618810195' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-13T00:52:50.937 INFO:teuthology.orchestra.run.smithi077.stdout:{"epoch":39,"flags":0,"active_gid":25304,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6800","nonce":1098489877},{"type":"v1","addr":"172.21.15.77:6801","nonce":1098489877}]},"active_addr":"172.21.15.77:6801/1098489877","active_change":"2024-09-13T00:38:58.290737+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24727,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.77:8443/","prometheus":"http://172.21.15.77:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":290,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.77:0","nonce":2808661874}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.77:0","nonce":556480035}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.77:0","nonce":501456484}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.77:0","nonce":2291918349}]}]} 2024-09-13T00:52:50.941 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-13T00:52:50.941 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-13T00:52:50.942 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T00:52:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:52 smithi088 bash[68960]: cluster 2024-09-13T00:52:50.868302+0000 mgr.y (mgr.25304) 509 : cluster [DBG] pgmap v1085: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:52.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:52:52 smithi088 bash[61791]: ts=2024-09-13T00:52:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:52:52.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:52 smithi077 bash[80681]: cluster 2024-09-13T00:52:50.868302+0000 mgr.y (mgr.25304) 509 : cluster [DBG] pgmap v1085: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:52.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:52 smithi077 bash[83127]: cluster 2024-09-13T00:52:50.868302+0000 mgr.y (mgr.25304) 509 : cluster [DBG] pgmap v1085: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:54 smithi088 bash[68960]: cluster 2024-09-13T00:52:52.869034+0000 mgr.y (mgr.25304) 510 : cluster [DBG] pgmap v1086: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:54.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:54 smithi077 bash[80681]: cluster 2024-09-13T00:52:52.869034+0000 mgr.y (mgr.25304) 510 : cluster [DBG] pgmap v1086: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:54.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:54 smithi077 bash[83127]: cluster 2024-09-13T00:52:52.869034+0000 mgr.y (mgr.25304) 510 : cluster [DBG] pgmap v1086: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:55.718 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:52:56.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:56 smithi088 bash[68960]: cluster 2024-09-13T00:52:54.870143+0000 mgr.y (mgr.25304) 511 : cluster [DBG] pgmap v1087: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:56.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:56 smithi077 bash[80681]: cluster 2024-09-13T00:52:54.870143+0000 mgr.y (mgr.25304) 511 : cluster [DBG] pgmap v1087: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:56.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:56 smithi077 bash[83127]: cluster 2024-09-13T00:52:54.870143+0000 mgr.y (mgr.25304) 511 : cluster [DBG] pgmap v1087: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:58.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:58 smithi077 bash[83127]: cluster 2024-09-13T00:52:56.870905+0000 mgr.y (mgr.25304) 512 : cluster [DBG] pgmap v1088: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:58.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:58 smithi077 bash[80681]: cluster 2024-09-13T00:52:56.870905+0000 mgr.y (mgr.25304) 512 : cluster [DBG] pgmap v1088: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:58.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:58 smithi088 bash[68960]: cluster 2024-09-13T00:52:56.870905+0000 mgr.y (mgr.25304) 512 : cluster [DBG] pgmap v1088: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:52:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:52:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:52:58] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-09-13T00:52:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:52:59 smithi077 bash[80681]: audit 2024-09-13T00:52:58.429369+0000 mon.a (mon.0) 1507 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:52:59.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:52:59 smithi077 bash[83127]: audit 2024-09-13T00:52:58.429369+0000 mon.a (mon.0) 1507 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:52:59.594 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:52:59.594 INFO:teuthology.orchestra.run.smithi077.stdout:{"epoch":956,"fsid":"9218306c-7165-11ef-bceb-c7b262605968","created":"2024-09-13T00:18:59.375829+0000","modified":"2024-09-13T00:52:40.252383+0000","last_up_change":"2024-09-13T00:46:44.420668+0000","last_in_change":"2024-09-13T00:28:05.097746+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":40,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T00:24:15.726444+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T00:30:09.880408+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":57,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T00:32:21.904347+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T00:32:22.489687+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e0860feb-72c9-4c49-933d-0a49b68caeee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":484,"up_from":502,"up_thru":904,"down_at":485,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6802","nonce":3174548933},{"type":"v1","addr":"172.21.15.77:6803","nonce":3174548933}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6804","nonce":3174548933},{"type":"v1","addr":"172.21.15.77:6805","nonce":3174548933}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6808","nonce":3174548933},{"type":"v1","addr":"172.21.15.77:6809","nonce":3174548933}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6806","nonce":3174548933},{"type":"v1","addr":"172.21.15.77:6807","nonce":3174548933}]},"public_addr":"172.21.15.77:6803/3174548933","cluster_addr":"172.21.15.77:6805/3174548933","heartbeat_back_addr":"172.21.15.77:6809/3174548933","heartbeat_front_addr":"172.21.15.77:6807/3174548933","state":["exists","up"]},{"osd":1,"uuid":"7ee01388-61b0-4a60-81f2-08c3c072ec0e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":522,"up_from":539,"up_thru":904,"down_at":523,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6810","nonce":975728159},{"type":"v1","addr":"172.21.15.77:6811","nonce":975728159}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6812","nonce":975728159},{"type":"v1","addr":"172.21.15.77:6813","nonce":975728159}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6816","nonce":975728159},{"type":"v1","addr":"172.21.15.77:6817","nonce":975728159}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6814","nonce":975728159},{"type":"v1","addr":"172.21.15.77:6815","nonce":975728159}]},"public_addr":"172.21.15.77:6811/975728159","cluster_addr":"172.21.15.77:6813/975728159","heartbeat_back_addr":"172.21.15.77:6817/975728159","heartbeat_front_addr":"172.21.15.77:6815/975728159","state":["exists","up"]},{"osd":2,"uuid":"ff8bc88a-5b11-469f-912d-d3149036b450","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":622,"up_from":629,"up_thru":904,"down_at":623,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6818","nonce":3472170327},{"type":"v1","addr":"172.21.15.77:6819","nonce":3472170327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6820","nonce":3472170327},{"type":"v1","addr":"172.21.15.77:6821","nonce":3472170327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6824","nonce":3472170327},{"type":"v1","addr":"172.21.15.77:6825","nonce":3472170327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6822","nonce":3472170327},{"type":"v1","addr":"172.21.15.77:6823","nonce":3472170327}]},"public_addr":"172.21.15.77:6819/3472170327","cluster_addr":"172.21.15.77:6821/3472170327","heartbeat_back_addr":"172.21.15.77:6825/3472170327","heartbeat_front_addr":"172.21.15.77:6823/3472170327","state":["exists","up"]},{"osd":3,"uuid":"cb032137-609e-41f6-bfc7-e70b77ea9456","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":637,"up_from":643,"up_thru":904,"down_at":637,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6826","nonce":2452864905},{"type":"v1","addr":"172.21.15.77:6827","nonce":2452864905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6828","nonce":2452864905},{"type":"v1","addr":"172.21.15.77:6829","nonce":2452864905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6832","nonce":2452864905},{"type":"v1","addr":"172.21.15.77:6833","nonce":2452864905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6830","nonce":2452864905},{"type":"v1","addr":"172.21.15.77:6831","nonce":2452864905}]},"public_addr":"172.21.15.77:6827/2452864905","cluster_addr":"172.21.15.77:6829/2452864905","heartbeat_back_addr":"172.21.15.77:6833/2452864905","heartbeat_front_addr":"172.21.15.77:6831/2452864905","state":["exists","up"]},{"osd":4,"uuid":"cd38603e-5cdc-42b6-b8c8-656d44df0ed5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":645,"up_from":649,"up_thru":904,"down_at":646,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6800","nonce":808774429},{"type":"v1","addr":"172.21.15.88:6801","nonce":808774429}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6802","nonce":808774429},{"type":"v1","addr":"172.21.15.88:6803","nonce":808774429}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6806","nonce":808774429},{"type":"v1","addr":"172.21.15.88:6807","nonce":808774429}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6804","nonce":808774429},{"type":"v1","addr":"172.21.15.88:6805","nonce":808774429}]},"public_addr":"172.21.15.88:6801/808774429","cluster_addr":"172.21.15.88:6803/808774429","heartbeat_back_addr":"172.21.15.88:6807/808774429","heartbeat_front_addr":"172.21.15.88:6805/808774429","state":["exists","up"]},{"osd":5,"uuid":"d4b81ff3-f257-4f63-a7a7-cbd1ff754544","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":904,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6808","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6809","nonce":2699666173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6810","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6811","nonce":2699666173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6814","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6815","nonce":2699666173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6812","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6813","nonce":2699666173}]},"public_addr":"172.21.15.88:6809/2699666173","cluster_addr":"172.21.15.88:6811/2699666173","heartbeat_back_addr":"172.21.15.88:6815/2699666173","heartbeat_front_addr":"172.21.15.88:6813/2699666173","state":["exists","up"]},{"osd":6,"uuid":"31c0d169-c329-440d-848b-057fbbf418ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":904,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6816","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6817","nonce":412460705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6818","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6819","nonce":412460705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6822","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6823","nonce":412460705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6820","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6821","nonce":412460705}]},"public_addr":"172.21.15.88:6817/412460705","cluster_addr":"172.21.15.88:6819/412460705","heartbeat_back_addr":"172.21.15.88:6823/412460705","heartbeat_front_addr":"172.21.15.88:6821/412460705","state":["exists","up"]},{"osd":7,"uuid":"2b2c1a7c-3203-4e9a-b07b-4eaae7e09c0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":904,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6824","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6825","nonce":2009347492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6826","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6827","nonce":2009347492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6830","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6831","nonce":2009347492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6828","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6829","nonce":2009347492}]},"public_addr":"172.21.15.88:6825/2009347492","cluster_addr":"172.21.15.88:6827/2009347492","heartbeat_back_addr":"172.21.15.88:6831/2009347492","heartbeat_front_addr":"172.21.15.88:6829/2009347492","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:22:21.643909+0000","dead_epoch":484},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:23:17.132405+0000","dead_epoch":522},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:24:13.726010+0000","dead_epoch":622},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:25:11.211228+0000","dead_epoch":637},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:26:00.949032+0000","dead_epoch":645},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:26:49.596019+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:27:38.694576+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:28:28.383054+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.9","mappings":[{"from":1,"to":5}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.77:0/4170359499":"2024-09-13T01:48:45.381376+0000","172.21.15.77:0/3073421762":"2024-09-13T01:43:45.051582+0000","172.21.15.77:0/2411268082":"2024-09-14T00:38:58.290178+0000","172.21.15.77:0/1647054719":"2024-09-14T00:38:58.290178+0000","172.21.15.77:0/120241471":"2024-09-14T00:38:58.290178+0000","172.21.15.77:6801/2126430742":"2024-09-14T00:38:58.290178+0000","172.21.15.77:6800/2126430742":"2024-09-14T00:38:58.290178+0000","172.21.15.77:0/449115915":"2024-09-14T00:38:58.290178+0000","172.21.15.77:6800/3213372405":"2024-09-14T00:19:55.144762+0000","172.21.15.88:0/3571369577":"2024-09-14T00:37:56.239850+0000","172.21.15.77:0/3515285019":"2024-09-14T00:36:50.768115+0000","172.21.15.88:0/678673763":"2024-09-14T00:37:56.239850+0000","172.21.15.77:0/2198980039":"2024-09-14T00:19:55.144762+0000","172.21.15.77:6801/930711876":"2024-09-14T00:36:50.768115+0000","172.21.15.77:0/1338001532":"2024-09-13T01:43:46.001799+0000","172.21.15.77:6800/426595304":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6801/426595304":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6800/395417143":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/3315541396":"2024-09-14T00:38:58.290178+0000","172.21.15.88:0/4261992538":"2024-09-14T00:37:56.239850+0000","172.21.15.77:0/1137737372":"2024-09-14T00:19:25.064947+0000","172.21.15.77:0/3827884320":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/1177255174":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/3006990786":"2024-09-14T00:38:58.290178+0000","172.21.15.77:0/4117010497":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/1897411349":"2024-09-14T00:19:55.144762+0000","172.21.15.77:6801/3213372405":"2024-09-14T00:19:55.144762+0000","172.21.15.88:0/372939794":"2024-09-14T00:37:56.239850+0000","172.21.15.77:6835/506597872":"2024-09-14T00:32:30.861677+0000","172.21.15.77:6801/395417143":"2024-09-14T00:29:27.728349+0000","172.21.15.88:0/2059234201":"2024-09-14T00:37:56.239850+0000","172.21.15.77:6800/930711876":"2024-09-14T00:36:50.768115+0000","172.21.15.77:0/1115799687":"2024-09-14T00:36:50.768115+0000","172.21.15.77:0/892433806":"2024-09-14T00:19:25.064947+0000","172.21.15.88:0/3372897986":"2024-09-14T00:37:56.239850+0000","172.21.15.77:0/2219740569":"2024-09-14T00:36:50.768115+0000","172.21.15.77:0/1347196058":"2024-09-14T00:29:27.728349+0000","172.21.15.88:6834/2960785315":"2024-09-14T00:37:56.239850+0000","172.21.15.88:6835/2960785315":"2024-09-14T00:37:56.239850+0000","172.21.15.77:0/4291417054":"2024-09-14T00:19:25.064947+0000","172.21.15.77:0/1008984793":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/676410089":"2024-09-14T00:36:50.768115+0000","172.21.15.77:6834/506597872":"2024-09-14T00:32:30.861677+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":13,"snaps":[{"begin":12,"length":1},{"begin":14,"length":1}]},{"pool":15,"snaps":[{"begin":61,"length":3},{"begin":69,"length":1},{"begin":71,"length":3}]},{"pool":16,"snaps":[{"begin":24,"length":1},{"begin":31,"length":1}]},{"pool":17,"snaps":[{"begin":2,"length":1}]},{"pool":40,"snaps":[{"begin":9,"length":3}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T00:52:59.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:52:59 smithi088 bash[68960]: audit 2024-09-13T00:52:58.429369+0000 mon.a (mon.0) 1507 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:53:00.391 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-13T00:53:00.391 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-13T00:53:00.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:00 smithi088 bash[68960]: cluster 2024-09-13T00:52:58.871688+0000 mgr.y (mgr.25304) 513 : cluster [DBG] pgmap v1089: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:00.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:00 smithi088 bash[68960]: audit 2024-09-13T00:52:59.593615+0000 mon.a (mon.0) 1508 : audit [DBG] from='client.? 172.21.15.77:0/4076177563' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:53:00.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:00 smithi077 bash[80681]: cluster 2024-09-13T00:52:58.871688+0000 mgr.y (mgr.25304) 513 : cluster [DBG] pgmap v1089: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:00.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:00 smithi077 bash[80681]: audit 2024-09-13T00:52:59.593615+0000 mon.a (mon.0) 1508 : audit [DBG] from='client.? 172.21.15.77:0/4076177563' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:53:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:00 smithi077 bash[83127]: cluster 2024-09-13T00:52:58.871688+0000 mgr.y (mgr.25304) 513 : cluster [DBG] pgmap v1089: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:00 smithi077 bash[83127]: audit 2024-09-13T00:52:59.593615+0000 mon.a (mon.0) 1508 : audit [DBG] from='client.? 172.21.15.77:0/4076177563' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:53:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:02 smithi088 bash[68960]: cluster 2024-09-13T00:53:00.872650+0000 mgr.y (mgr.25304) 514 : cluster [DBG] pgmap v1090: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:02.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:53:02 smithi088 bash[61791]: ts=2024-09-13T00:53:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:53:02.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:02 smithi077 bash[80681]: cluster 2024-09-13T00:53:00.872650+0000 mgr.y (mgr.25304) 514 : cluster [DBG] pgmap v1090: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:02.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:02 smithi077 bash[83127]: cluster 2024-09-13T00:53:00.872650+0000 mgr.y (mgr.25304) 514 : cluster [DBG] pgmap v1090: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:04.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:04 smithi088 bash[68960]: cluster 2024-09-13T00:53:02.873422+0000 mgr.y (mgr.25304) 515 : cluster [DBG] pgmap v1091: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:04.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:04 smithi077 bash[80681]: cluster 2024-09-13T00:53:02.873422+0000 mgr.y (mgr.25304) 515 : cluster [DBG] pgmap v1091: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:04.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:04 smithi077 bash[83127]: cluster 2024-09-13T00:53:02.873422+0000 mgr.y (mgr.25304) 515 : cluster [DBG] pgmap v1091: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:05.174 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:06.584 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:06 smithi077 bash[80681]: cluster 2024-09-13T00:53:04.874450+0000 mgr.y (mgr.25304) 516 : cluster [DBG] pgmap v1092: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:06.584 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:06 smithi077 bash[83127]: cluster 2024-09-13T00:53:04.874450+0000 mgr.y (mgr.25304) 516 : cluster [DBG] pgmap v1092: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:06.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:06 smithi088 bash[68960]: cluster 2024-09-13T00:53:04.874450+0000 mgr.y (mgr.25304) 516 : cluster [DBG] pgmap v1092: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:08.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:08 smithi088 bash[68960]: cluster 2024-09-13T00:53:06.875015+0000 mgr.y (mgr.25304) 517 : cluster [DBG] pgmap v1093: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:08.668 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:08 smithi077 bash[80681]: cluster 2024-09-13T00:53:06.875015+0000 mgr.y (mgr.25304) 517 : cluster [DBG] pgmap v1093: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:08.668 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:53:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:53:08] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-09-13T00:53:08.669 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:08 smithi077 bash[83127]: cluster 2024-09-13T00:53:06.875015+0000 mgr.y (mgr.25304) 517 : cluster [DBG] pgmap v1093: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:08.997 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:53:08.998 INFO:teuthology.orchestra.run.smithi077.stdout:{"epoch":956,"fsid":"9218306c-7165-11ef-bceb-c7b262605968","created":"2024-09-13T00:18:59.375829+0000","modified":"2024-09-13T00:52:40.252383+0000","last_up_change":"2024-09-13T00:46:44.420668+0000","last_in_change":"2024-09-13T00:28:05.097746+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":40,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-13T00:24:15.726444+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-13T00:30:09.880408+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"57","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":57,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-13T00:32:21.904347+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-13T00:32:22.489687+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"e0860feb-72c9-4c49-933d-0a49b68caeee","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":484,"up_from":502,"up_thru":904,"down_at":485,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6802","nonce":3174548933},{"type":"v1","addr":"172.21.15.77:6803","nonce":3174548933}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6804","nonce":3174548933},{"type":"v1","addr":"172.21.15.77:6805","nonce":3174548933}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6808","nonce":3174548933},{"type":"v1","addr":"172.21.15.77:6809","nonce":3174548933}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6806","nonce":3174548933},{"type":"v1","addr":"172.21.15.77:6807","nonce":3174548933}]},"public_addr":"172.21.15.77:6803/3174548933","cluster_addr":"172.21.15.77:6805/3174548933","heartbeat_back_addr":"172.21.15.77:6809/3174548933","heartbeat_front_addr":"172.21.15.77:6807/3174548933","state":["exists","up"]},{"osd":1,"uuid":"7ee01388-61b0-4a60-81f2-08c3c072ec0e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":522,"up_from":539,"up_thru":904,"down_at":523,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6810","nonce":975728159},{"type":"v1","addr":"172.21.15.77:6811","nonce":975728159}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6812","nonce":975728159},{"type":"v1","addr":"172.21.15.77:6813","nonce":975728159}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6816","nonce":975728159},{"type":"v1","addr":"172.21.15.77:6817","nonce":975728159}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6814","nonce":975728159},{"type":"v1","addr":"172.21.15.77:6815","nonce":975728159}]},"public_addr":"172.21.15.77:6811/975728159","cluster_addr":"172.21.15.77:6813/975728159","heartbeat_back_addr":"172.21.15.77:6817/975728159","heartbeat_front_addr":"172.21.15.77:6815/975728159","state":["exists","up"]},{"osd":2,"uuid":"ff8bc88a-5b11-469f-912d-d3149036b450","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":622,"up_from":629,"up_thru":904,"down_at":623,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6818","nonce":3472170327},{"type":"v1","addr":"172.21.15.77:6819","nonce":3472170327}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6820","nonce":3472170327},{"type":"v1","addr":"172.21.15.77:6821","nonce":3472170327}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6824","nonce":3472170327},{"type":"v1","addr":"172.21.15.77:6825","nonce":3472170327}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6822","nonce":3472170327},{"type":"v1","addr":"172.21.15.77:6823","nonce":3472170327}]},"public_addr":"172.21.15.77:6819/3472170327","cluster_addr":"172.21.15.77:6821/3472170327","heartbeat_back_addr":"172.21.15.77:6825/3472170327","heartbeat_front_addr":"172.21.15.77:6823/3472170327","state":["exists","up"]},{"osd":3,"uuid":"cb032137-609e-41f6-bfc7-e70b77ea9456","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":637,"up_from":643,"up_thru":904,"down_at":637,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6826","nonce":2452864905},{"type":"v1","addr":"172.21.15.77:6827","nonce":2452864905}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6828","nonce":2452864905},{"type":"v1","addr":"172.21.15.77:6829","nonce":2452864905}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6832","nonce":2452864905},{"type":"v1","addr":"172.21.15.77:6833","nonce":2452864905}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.77:6830","nonce":2452864905},{"type":"v1","addr":"172.21.15.77:6831","nonce":2452864905}]},"public_addr":"172.21.15.77:6827/2452864905","cluster_addr":"172.21.15.77:6829/2452864905","heartbeat_back_addr":"172.21.15.77:6833/2452864905","heartbeat_front_addr":"172.21.15.77:6831/2452864905","state":["exists","up"]},{"osd":4,"uuid":"cd38603e-5cdc-42b6-b8c8-656d44df0ed5","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":645,"up_from":649,"up_thru":904,"down_at":646,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6800","nonce":808774429},{"type":"v1","addr":"172.21.15.88:6801","nonce":808774429}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6802","nonce":808774429},{"type":"v1","addr":"172.21.15.88:6803","nonce":808774429}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6806","nonce":808774429},{"type":"v1","addr":"172.21.15.88:6807","nonce":808774429}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6804","nonce":808774429},{"type":"v1","addr":"172.21.15.88:6805","nonce":808774429}]},"public_addr":"172.21.15.88:6801/808774429","cluster_addr":"172.21.15.88:6803/808774429","heartbeat_back_addr":"172.21.15.88:6807/808774429","heartbeat_front_addr":"172.21.15.88:6805/808774429","state":["exists","up"]},{"osd":5,"uuid":"d4b81ff3-f257-4f63-a7a7-cbd1ff754544","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":36,"up_thru":904,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6808","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6809","nonce":2699666173}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6810","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6811","nonce":2699666173}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6814","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6815","nonce":2699666173}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6812","nonce":2699666173},{"type":"v1","addr":"172.21.15.88:6813","nonce":2699666173}]},"public_addr":"172.21.15.88:6809/2699666173","cluster_addr":"172.21.15.88:6811/2699666173","heartbeat_back_addr":"172.21.15.88:6815/2699666173","heartbeat_front_addr":"172.21.15.88:6813/2699666173","state":["exists","up"]},{"osd":6,"uuid":"31c0d169-c329-440d-848b-057fbbf418ca","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":42,"up_thru":904,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6816","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6817","nonce":412460705}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6818","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6819","nonce":412460705}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6822","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6823","nonce":412460705}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6820","nonce":412460705},{"type":"v1","addr":"172.21.15.88:6821","nonce":412460705}]},"public_addr":"172.21.15.88:6817/412460705","cluster_addr":"172.21.15.88:6819/412460705","heartbeat_back_addr":"172.21.15.88:6823/412460705","heartbeat_front_addr":"172.21.15.88:6821/412460705","state":["exists","up"]},{"osd":7,"uuid":"2b2c1a7c-3203-4e9a-b07b-4eaae7e09c0c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":48,"up_thru":904,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6824","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6825","nonce":2009347492}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6826","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6827","nonce":2009347492}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6830","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6831","nonce":2009347492}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.88:6828","nonce":2009347492},{"type":"v1","addr":"172.21.15.88:6829","nonce":2009347492}]},"public_addr":"172.21.15.88:6825/2009347492","cluster_addr":"172.21.15.88:6827/2009347492","heartbeat_back_addr":"172.21.15.88:6831/2009347492","heartbeat_front_addr":"172.21.15.88:6829/2009347492","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:22:21.643909+0000","dead_epoch":484},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:23:17.132405+0000","dead_epoch":522},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:24:13.726010+0000","dead_epoch":622},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:25:11.211228+0000","dead_epoch":637},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:26:00.949032+0000","dead_epoch":645},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:26:49.596019+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:27:38.694576+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-09-13T00:28:28.383054+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.9","mappings":[{"from":1,"to":5}]},{"pgid":"4.b","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.77:0/4170359499":"2024-09-13T01:48:45.381376+0000","172.21.15.77:0/3073421762":"2024-09-13T01:43:45.051582+0000","172.21.15.77:0/2411268082":"2024-09-14T00:38:58.290178+0000","172.21.15.77:0/1647054719":"2024-09-14T00:38:58.290178+0000","172.21.15.77:0/120241471":"2024-09-14T00:38:58.290178+0000","172.21.15.77:6801/2126430742":"2024-09-14T00:38:58.290178+0000","172.21.15.77:6800/2126430742":"2024-09-14T00:38:58.290178+0000","172.21.15.77:0/449115915":"2024-09-14T00:38:58.290178+0000","172.21.15.77:6800/3213372405":"2024-09-14T00:19:55.144762+0000","172.21.15.88:0/3571369577":"2024-09-14T00:37:56.239850+0000","172.21.15.77:0/3515285019":"2024-09-14T00:36:50.768115+0000","172.21.15.88:0/678673763":"2024-09-14T00:37:56.239850+0000","172.21.15.77:0/2198980039":"2024-09-14T00:19:55.144762+0000","172.21.15.77:6801/930711876":"2024-09-14T00:36:50.768115+0000","172.21.15.77:0/1338001532":"2024-09-13T01:43:46.001799+0000","172.21.15.77:6800/426595304":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6801/426595304":"2024-09-14T00:19:25.064947+0000","172.21.15.77:6800/395417143":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/3315541396":"2024-09-14T00:38:58.290178+0000","172.21.15.88:0/4261992538":"2024-09-14T00:37:56.239850+0000","172.21.15.77:0/1137737372":"2024-09-14T00:19:25.064947+0000","172.21.15.77:0/3827884320":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/1177255174":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/3006990786":"2024-09-14T00:38:58.290178+0000","172.21.15.77:0/4117010497":"2024-09-14T00:19:55.144762+0000","172.21.15.77:0/1897411349":"2024-09-14T00:19:55.144762+0000","172.21.15.77:6801/3213372405":"2024-09-14T00:19:55.144762+0000","172.21.15.88:0/372939794":"2024-09-14T00:37:56.239850+0000","172.21.15.77:6835/506597872":"2024-09-14T00:32:30.861677+0000","172.21.15.77:6801/395417143":"2024-09-14T00:29:27.728349+0000","172.21.15.88:0/2059234201":"2024-09-14T00:37:56.239850+0000","172.21.15.77:6800/930711876":"2024-09-14T00:36:50.768115+0000","172.21.15.77:0/1115799687":"2024-09-14T00:36:50.768115+0000","172.21.15.77:0/892433806":"2024-09-14T00:19:25.064947+0000","172.21.15.88:0/3372897986":"2024-09-14T00:37:56.239850+0000","172.21.15.77:0/2219740569":"2024-09-14T00:36:50.768115+0000","172.21.15.77:0/1347196058":"2024-09-14T00:29:27.728349+0000","172.21.15.88:6834/2960785315":"2024-09-14T00:37:56.239850+0000","172.21.15.88:6835/2960785315":"2024-09-14T00:37:56.239850+0000","172.21.15.77:0/4291417054":"2024-09-14T00:19:25.064947+0000","172.21.15.77:0/1008984793":"2024-09-14T00:29:27.728349+0000","172.21.15.77:0/676410089":"2024-09-14T00:36:50.768115+0000","172.21.15.77:6834/506597872":"2024-09-14T00:32:30.861677+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":13,"snaps":[{"begin":12,"length":1},{"begin":14,"length":1}]},{"pool":15,"snaps":[{"begin":61,"length":3},{"begin":69,"length":1},{"begin":71,"length":3}]},{"pool":16,"snaps":[{"begin":24,"length":1},{"begin":31,"length":1}]},{"pool":17,"snaps":[{"begin":2,"length":1}]},{"pool":40,"snaps":[{"begin":9,"length":3}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-13T00:53:09.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:09 smithi077 bash[80681]: audit 2024-09-13T00:53:08.998886+0000 mon.a (mon.0) 1509 : audit [DBG] from='client.? 172.21.15.77:0/1828034750' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:53:09.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:09 smithi077 bash[83127]: audit 2024-09-13T00:53:08.998886+0000 mon.a (mon.0) 1509 : audit [DBG] from='client.? 172.21.15.77:0/1828034750' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:53:09.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:09 smithi088 bash[68960]: audit 2024-09-13T00:53:08.998886+0000 mon.a (mon.0) 1509 : audit [DBG] from='client.? 172.21.15.77:0/1828034750' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-13T00:53:09.871 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-13T00:53:09.872 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-13T00:53:09.872 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-13T00:53:09.873 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-13T00:53:09.873 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-13T00:53:09.874 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-13T00:53:09.874 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-13T00:53:09.875 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-13T00:53:10.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:10 smithi088 bash[68960]: cluster 2024-09-13T00:53:08.875769+0000 mgr.y (mgr.25304) 518 : cluster [DBG] pgmap v1094: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:10.781 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:10 smithi077 bash[80681]: cluster 2024-09-13T00:53:08.875769+0000 mgr.y (mgr.25304) 518 : cluster [DBG] pgmap v1094: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:10.781 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:10 smithi077 bash[83127]: cluster 2024-09-13T00:53:08.875769+0000 mgr.y (mgr.25304) 518 : cluster [DBG] pgmap v1094: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:11.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:11 smithi077 bash[80681]: audit 2024-09-13T00:53:10.846671+0000 mon.a (mon.0) 1510 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:53:11.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:11 smithi077 bash[83127]: audit 2024-09-13T00:53:10.846671+0000 mon.a (mon.0) 1510 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:53:11.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:11 smithi088 bash[68960]: audit 2024-09-13T00:53:10.846671+0000 mon.a (mon.0) 1510 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:53:12.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:12 smithi088 bash[68960]: cluster 2024-09-13T00:53:10.876660+0000 mgr.y (mgr.25304) 519 : cluster [DBG] pgmap v1095: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:12.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:53:12 smithi088 bash[61791]: ts=2024-09-13T00:53:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:53:12.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:12 smithi077 bash[80681]: cluster 2024-09-13T00:53:10.876660+0000 mgr.y (mgr.25304) 519 : cluster [DBG] pgmap v1095: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:12.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:12 smithi077 bash[83127]: cluster 2024-09-13T00:53:10.876660+0000 mgr.y (mgr.25304) 519 : cluster [DBG] pgmap v1095: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:14.771 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:14.772 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:14.772 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:14.772 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:14.774 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:14.774 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:14.774 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:14.775 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:14.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:14 smithi077 bash[83127]: cluster 2024-09-13T00:53:12.877373+0000 mgr.y (mgr.25304) 520 : cluster [DBG] pgmap v1096: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:14.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:14 smithi077 bash[83127]: audit 2024-09-13T00:53:13.429521+0000 mon.a (mon.0) 1511 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:53:14.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:14 smithi077 bash[80681]: cluster 2024-09-13T00:53:12.877373+0000 mgr.y (mgr.25304) 520 : cluster [DBG] pgmap v1096: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:14.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:14 smithi077 bash[80681]: audit 2024-09-13T00:53:13.429521+0000 mon.a (mon.0) 1511 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:53:14.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:14 smithi088 bash[68960]: cluster 2024-09-13T00:53:12.877373+0000 mgr.y (mgr.25304) 520 : cluster [DBG] pgmap v1096: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:14.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:14 smithi088 bash[68960]: audit 2024-09-13T00:53:13.429521+0000 mon.a (mon.0) 1511 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:53:16.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:16 smithi077 bash[80681]: cluster 2024-09-13T00:53:14.878130+0000 mgr.y (mgr.25304) 521 : cluster [DBG] pgmap v1097: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:16.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:16 smithi077 bash[80681]: audit 2024-09-13T00:53:15.719012+0000 mon.a (mon.0) 1512 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:53:16.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:16 smithi077 bash[80681]: audit 2024-09-13T00:53:15.724961+0000 mon.a (mon.0) 1513 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:53:16.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:16 smithi077 bash[83127]: cluster 2024-09-13T00:53:14.878130+0000 mgr.y (mgr.25304) 521 : cluster [DBG] pgmap v1097: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:16.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:16 smithi077 bash[83127]: audit 2024-09-13T00:53:15.719012+0000 mon.a (mon.0) 1512 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:53:16.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:16 smithi077 bash[83127]: audit 2024-09-13T00:53:15.724961+0000 mon.a (mon.0) 1513 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:53:16.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:16 smithi088 bash[68960]: cluster 2024-09-13T00:53:14.878130+0000 mgr.y (mgr.25304) 521 : cluster [DBG] pgmap v1097: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:16.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:16 smithi088 bash[68960]: audit 2024-09-13T00:53:15.719012+0000 mon.a (mon.0) 1512 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:53:16.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:16 smithi088 bash[68960]: audit 2024-09-13T00:53:15.724961+0000 mon.a (mon.0) 1513 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:53:18.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:17 smithi077 bash[80681]: audit 2024-09-13T00:53:16.656434+0000 mon.a (mon.0) 1514 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:53:18.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:17 smithi077 bash[80681]: audit 2024-09-13T00:53:16.668323+0000 mon.a (mon.0) 1515 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:53:18.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:17 smithi077 bash[80681]: cluster 2024-09-13T00:53:16.879013+0000 mgr.y (mgr.25304) 522 : cluster [DBG] pgmap v1098: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:18.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:17 smithi077 bash[80681]: audit 2024-09-13T00:53:17.128935+0000 mon.a (mon.0) 1516 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:53:18.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:17 smithi077 bash[80681]: audit 2024-09-13T00:53:17.130658+0000 mon.a (mon.0) 1517 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:53:18.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:17 smithi077 bash[80681]: audit 2024-09-13T00:53:17.140375+0000 mon.a (mon.0) 1518 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:53:18.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:17 smithi077 bash[83127]: audit 2024-09-13T00:53:16.656434+0000 mon.a (mon.0) 1514 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:53:18.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:17 smithi077 bash[83127]: audit 2024-09-13T00:53:16.668323+0000 mon.a (mon.0) 1515 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:53:18.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:17 smithi077 bash[83127]: cluster 2024-09-13T00:53:16.879013+0000 mgr.y (mgr.25304) 522 : cluster [DBG] pgmap v1098: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:18.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:17 smithi077 bash[83127]: audit 2024-09-13T00:53:17.128935+0000 mon.a (mon.0) 1516 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:53:18.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:17 smithi077 bash[83127]: audit 2024-09-13T00:53:17.130658+0000 mon.a (mon.0) 1517 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:53:18.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:17 smithi077 bash[83127]: audit 2024-09-13T00:53:17.140375+0000 mon.a (mon.0) 1518 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:53:18.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:17 smithi088 bash[68960]: audit 2024-09-13T00:53:16.656434+0000 mon.a (mon.0) 1514 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:53:18.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:17 smithi088 bash[68960]: audit 2024-09-13T00:53:16.668323+0000 mon.a (mon.0) 1515 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:53:18.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:17 smithi088 bash[68960]: cluster 2024-09-13T00:53:16.879013+0000 mgr.y (mgr.25304) 522 : cluster [DBG] pgmap v1098: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:18.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:17 smithi088 bash[68960]: audit 2024-09-13T00:53:17.128935+0000 mon.a (mon.0) 1516 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:53:18.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:17 smithi088 bash[68960]: audit 2024-09-13T00:53:17.130658+0000 mon.a (mon.0) 1517 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:53:18.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:17 smithi088 bash[68960]: audit 2024-09-13T00:53:17.140375+0000 mon.a (mon.0) 1518 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:53:18.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:53:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:53:18] "GET /metrics HTTP/1.1" 200 34266 "" "Prometheus/2.43.0" 2024-09-13T00:53:20.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:19 smithi077 bash[80681]: cluster 2024-09-13T00:53:18.879582+0000 mgr.y (mgr.25304) 523 : cluster [DBG] pgmap v1099: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:20.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:19 smithi077 bash[83127]: cluster 2024-09-13T00:53:18.879582+0000 mgr.y (mgr.25304) 523 : cluster [DBG] pgmap v1099: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:20.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:19 smithi088 bash[68960]: cluster 2024-09-13T00:53:18.879582+0000 mgr.y (mgr.25304) 523 : cluster [DBG] pgmap v1099: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:22.196 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:21 smithi077 bash[80681]: cluster 2024-09-13T00:53:20.880109+0000 mgr.y (mgr.25304) 524 : cluster [DBG] pgmap v1100: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:22.197 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:21 smithi077 bash[83127]: cluster 2024-09-13T00:53:20.880109+0000 mgr.y (mgr.25304) 524 : cluster [DBG] pgmap v1100: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:22.314 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:21 smithi088 bash[68960]: cluster 2024-09-13T00:53:20.880109+0000 mgr.y (mgr.25304) 524 : cluster [DBG] pgmap v1100: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:22.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:53:22 smithi088 bash[61791]: ts=2024-09-13T00:53:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:53:24.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:23 smithi077 bash[80681]: cluster 2024-09-13T00:53:22.880744+0000 mgr.y (mgr.25304) 525 : cluster [DBG] pgmap v1101: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:24.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:23 smithi077 bash[83127]: cluster 2024-09-13T00:53:22.880744+0000 mgr.y (mgr.25304) 525 : cluster [DBG] pgmap v1101: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:24.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:23 smithi088 bash[68960]: cluster 2024-09-13T00:53:22.880744+0000 mgr.y (mgr.25304) 525 : cluster [DBG] pgmap v1101: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:26.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:25 smithi077 bash[80681]: cluster 2024-09-13T00:53:24.881763+0000 mgr.y (mgr.25304) 526 : cluster [DBG] pgmap v1102: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:26.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:25 smithi077 bash[83127]: cluster 2024-09-13T00:53:24.881763+0000 mgr.y (mgr.25304) 526 : cluster [DBG] pgmap v1102: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:26.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:25 smithi088 bash[68960]: cluster 2024-09-13T00:53:24.881763+0000 mgr.y (mgr.25304) 526 : cluster [DBG] pgmap v1102: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:28.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:27 smithi077 bash[80681]: cluster 2024-09-13T00:53:26.882634+0000 mgr.y (mgr.25304) 527 : cluster [DBG] pgmap v1103: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:28.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:27 smithi077 bash[83127]: cluster 2024-09-13T00:53:26.882634+0000 mgr.y (mgr.25304) 527 : cluster [DBG] pgmap v1103: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:28.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:27 smithi088 bash[68960]: cluster 2024-09-13T00:53:26.882634+0000 mgr.y (mgr.25304) 527 : cluster [DBG] pgmap v1103: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:28.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:53:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:53:28] "GET /metrics HTTP/1.1" 200 34266 "" "Prometheus/2.43.0" 2024-09-13T00:53:29.311 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:29 smithi077 bash[83127]: audit 2024-09-13T00:53:28.430126+0000 mon.a (mon.0) 1519 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:53:29.311 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:28 smithi077 bash[80681]: audit 2024-09-13T00:53:28.430126+0000 mon.a (mon.0) 1519 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:53:29.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:28 smithi088 bash[68960]: audit 2024-09-13T00:53:28.430126+0000 mon.a (mon.0) 1519 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:53:30.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:29 smithi077 bash[80681]: cluster 2024-09-13T00:53:28.883507+0000 mgr.y (mgr.25304) 528 : cluster [DBG] pgmap v1104: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:30.028 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:29 smithi077 bash[83127]: cluster 2024-09-13T00:53:28.883507+0000 mgr.y (mgr.25304) 528 : cluster [DBG] pgmap v1104: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:30.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:29 smithi088 bash[68960]: cluster 2024-09-13T00:53:28.883507+0000 mgr.y (mgr.25304) 528 : cluster [DBG] pgmap v1104: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:32.246 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:31 smithi077 bash[80681]: cluster 2024-09-13T00:53:30.884412+0000 mgr.y (mgr.25304) 529 : cluster [DBG] pgmap v1105: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:32.247 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:31 smithi077 bash[83127]: cluster 2024-09-13T00:53:30.884412+0000 mgr.y (mgr.25304) 529 : cluster [DBG] pgmap v1105: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:32.314 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:31 smithi088 bash[68960]: cluster 2024-09-13T00:53:30.884412+0000 mgr.y (mgr.25304) 529 : cluster [DBG] pgmap v1105: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:32.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:53:32 smithi088 bash[61791]: ts=2024-09-13T00:53:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:53:33.509 INFO:teuthology.orchestra.run.smithi077.stdout:2156073582716 2024-09-13T00:53:33.510 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-13T00:53:34.143 INFO:teuthology.orchestra.run.smithi077.stdout:154618822980 2024-09-13T00:53:34.143 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-13T00:53:34.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:33 smithi077 bash[80681]: cluster 2024-09-13T00:53:32.885297+0000 mgr.y (mgr.25304) 530 : cluster [DBG] pgmap v1106: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:34.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:33 smithi077 bash[83127]: cluster 2024-09-13T00:53:32.885297+0000 mgr.y (mgr.25304) 530 : cluster [DBG] pgmap v1106: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:34.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:33 smithi088 bash[68960]: cluster 2024-09-13T00:53:32.885297+0000 mgr.y (mgr.25304) 530 : cluster [DBG] pgmap v1106: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:36.122 INFO:teuthology.orchestra.run.smithi077.stdout:2787433775187 2024-09-13T00:53:36.122 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-13T00:53:36.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:35 smithi077 bash[80681]: cluster 2024-09-13T00:53:34.886474+0000 mgr.y (mgr.25304) 531 : cluster [DBG] pgmap v1107: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:36.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:35 smithi077 bash[83127]: cluster 2024-09-13T00:53:34.886474+0000 mgr.y (mgr.25304) 531 : cluster [DBG] pgmap v1107: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:36.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:35 smithi088 bash[68960]: cluster 2024-09-13T00:53:34.886474+0000 mgr.y (mgr.25304) 531 : cluster [DBG] pgmap v1107: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:36.663 INFO:teuthology.orchestra.run.smithi077.stdout:2314987372661 2024-09-13T00:53:36.663 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-13T00:53:37.148 INFO:teuthology.orchestra.run.smithi077.stdout:2701534429280 2024-09-13T00:53:37.148 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-13T00:53:38.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:37 smithi077 bash[80681]: cluster 2024-09-13T00:53:36.887431+0000 mgr.y (mgr.25304) 532 : cluster [DBG] pgmap v1108: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:38.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:38 smithi077 bash[83127]: cluster 2024-09-13T00:53:36.887431+0000 mgr.y (mgr.25304) 532 : cluster [DBG] pgmap v1108: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:38.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:37 smithi088 bash[68960]: cluster 2024-09-13T00:53:36.887431+0000 mgr.y (mgr.25304) 532 : cluster [DBG] pgmap v1108: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:38.821 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:53:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:53:38] "GET /metrics HTTP/1.1" 200 34266 "" "Prometheus/2.43.0" 2024-09-13T00:53:39.314 INFO:teuthology.orchestra.run.smithi077.stdout:2761663971418 2024-09-13T00:53:39.315 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-13T00:53:39.524 INFO:teuthology.orchestra.run.smithi077.stdout:206158430513 2024-09-13T00:53:39.525 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-13T00:53:39.601 INFO:teuthology.orchestra.run.smithi077.stdout:180388626747 2024-09-13T00:53:39.601 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-13T00:53:40.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:39 smithi077 bash[80681]: cluster 2024-09-13T00:53:38.888471+0000 mgr.y (mgr.25304) 533 : cluster [DBG] pgmap v1109: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:40.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:39 smithi077 bash[83127]: cluster 2024-09-13T00:53:38.888471+0000 mgr.y (mgr.25304) 533 : cluster [DBG] pgmap v1109: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:40.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:39 smithi088 bash[68960]: cluster 2024-09-13T00:53:38.888471+0000 mgr.y (mgr.25304) 533 : cluster [DBG] pgmap v1109: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:41.721 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:41.722 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:41.724 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:41.724 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:41.725 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:42.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:41 smithi077 bash[80681]: cluster 2024-09-13T00:53:40.889409+0000 mgr.y (mgr.25304) 534 : cluster [DBG] pgmap v1110: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:42.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:42 smithi077 bash[83127]: cluster 2024-09-13T00:53:40.889409+0000 mgr.y (mgr.25304) 534 : cluster [DBG] pgmap v1110: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:42.314 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:41 smithi088 bash[68960]: cluster 2024-09-13T00:53:40.889409+0000 mgr.y (mgr.25304) 534 : cluster [DBG] pgmap v1110: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:42.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:53:42 smithi088 bash[61791]: ts=2024-09-13T00:53:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:53:43.762 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:43.763 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:43.764 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:53:44.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:43 smithi077 bash[83127]: cluster 2024-09-13T00:53:42.890079+0000 mgr.y (mgr.25304) 535 : cluster [DBG] pgmap v1111: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:44.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:43 smithi077 bash[83127]: audit 2024-09-13T00:53:43.430580+0000 mon.a (mon.0) 1520 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:53:44.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:43 smithi077 bash[80681]: cluster 2024-09-13T00:53:42.890079+0000 mgr.y (mgr.25304) 535 : cluster [DBG] pgmap v1111: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:44.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:43 smithi077 bash[80681]: audit 2024-09-13T00:53:43.430580+0000 mon.a (mon.0) 1520 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:53:44.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:43 smithi088 bash[68960]: cluster 2024-09-13T00:53:42.890079+0000 mgr.y (mgr.25304) 535 : cluster [DBG] pgmap v1111: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:44.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:43 smithi088 bash[68960]: audit 2024-09-13T00:53:43.430580+0000 mon.a (mon.0) 1520 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:53:46.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:45 smithi077 bash[80681]: cluster 2024-09-13T00:53:44.891187+0000 mgr.y (mgr.25304) 536 : cluster [DBG] pgmap v1112: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:46.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:45 smithi077 bash[83127]: cluster 2024-09-13T00:53:44.891187+0000 mgr.y (mgr.25304) 536 : cluster [DBG] pgmap v1112: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:46.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:45 smithi088 bash[68960]: cluster 2024-09-13T00:53:44.891187+0000 mgr.y (mgr.25304) 536 : cluster [DBG] pgmap v1112: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:48.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:47 smithi077 bash[80681]: cluster 2024-09-13T00:53:46.891934+0000 mgr.y (mgr.25304) 537 : cluster [DBG] pgmap v1113: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:48.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:47 smithi077 bash[83127]: cluster 2024-09-13T00:53:46.891934+0000 mgr.y (mgr.25304) 537 : cluster [DBG] pgmap v1113: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:48.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:47 smithi088 bash[68960]: cluster 2024-09-13T00:53:46.891934+0000 mgr.y (mgr.25304) 537 : cluster [DBG] pgmap v1113: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:48.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:53:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:53:48] "GET /metrics HTTP/1.1" 200 34262 "" "Prometheus/2.43.0" 2024-09-13T00:53:50.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:49 smithi077 bash[80681]: cluster 2024-09-13T00:53:48.892775+0000 mgr.y (mgr.25304) 538 : cluster [DBG] pgmap v1114: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:50.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:50 smithi077 bash[83127]: cluster 2024-09-13T00:53:48.892775+0000 mgr.y (mgr.25304) 538 : cluster [DBG] pgmap v1114: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:50.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:49 smithi088 bash[68960]: cluster 2024-09-13T00:53:48.892775+0000 mgr.y (mgr.25304) 538 : cluster [DBG] pgmap v1114: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:52.253 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:52 smithi077 bash[80681]: cluster 2024-09-13T00:53:50.893682+0000 mgr.y (mgr.25304) 539 : cluster [DBG] pgmap v1115: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:52.253 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:52 smithi077 bash[83127]: cluster 2024-09-13T00:53:50.893682+0000 mgr.y (mgr.25304) 539 : cluster [DBG] pgmap v1115: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:52.314 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:52 smithi088 bash[68960]: cluster 2024-09-13T00:53:50.893682+0000 mgr.y (mgr.25304) 539 : cluster [DBG] pgmap v1115: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:52.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:53:52 smithi088 bash[61791]: ts=2024-09-13T00:53:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:53:54.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:54 smithi077 bash[80681]: cluster 2024-09-13T00:53:52.894420+0000 mgr.y (mgr.25304) 540 : cluster [DBG] pgmap v1116: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:54.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:54 smithi077 bash[83127]: cluster 2024-09-13T00:53:52.894420+0000 mgr.y (mgr.25304) 540 : cluster [DBG] pgmap v1116: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:54.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:54 smithi088 bash[68960]: cluster 2024-09-13T00:53:52.894420+0000 mgr.y (mgr.25304) 540 : cluster [DBG] pgmap v1116: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:56.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:56 smithi088 bash[68960]: cluster 2024-09-13T00:53:54.895523+0000 mgr.y (mgr.25304) 541 : cluster [DBG] pgmap v1117: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:56 smithi077 bash[80681]: cluster 2024-09-13T00:53:54.895523+0000 mgr.y (mgr.25304) 541 : cluster [DBG] pgmap v1117: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:56 smithi077 bash[83127]: cluster 2024-09-13T00:53:54.895523+0000 mgr.y (mgr.25304) 541 : cluster [DBG] pgmap v1117: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:58.304 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:58 smithi077 bash[80681]: cluster 2024-09-13T00:53:56.896358+0000 mgr.y (mgr.25304) 542 : cluster [DBG] pgmap v1118: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:58.305 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:58 smithi077 bash[83127]: cluster 2024-09-13T00:53:56.896358+0000 mgr.y (mgr.25304) 542 : cluster [DBG] pgmap v1118: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:58.306 INFO:teuthology.orchestra.run.smithi077.stdout:2314987372666 2024-09-13T00:53:58.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:58 smithi088 bash[68960]: cluster 2024-09-13T00:53:56.896358+0000 mgr.y (mgr.25304) 542 : cluster [DBG] pgmap v1118: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:53:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:53:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:53:58] "GET /metrics HTTP/1.1" 200 34265 "" "Prometheus/2.43.0" 2024-09-13T00:53:59.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:59 smithi077 bash[80681]: audit 2024-09-13T00:53:58.309451+0000 mon.a (mon.0) 1521 : audit [DBG] from='client.? 172.21.15.77:0/27583279' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T00:53:59.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:53:59 smithi077 bash[80681]: audit 2024-09-13T00:53:58.430136+0000 mon.a (mon.0) 1522 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:53:59.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:59 smithi077 bash[83127]: audit 2024-09-13T00:53:58.309451+0000 mon.a (mon.0) 1521 : audit [DBG] from='client.? 172.21.15.77:0/27583279' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T00:53:59.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:53:59 smithi077 bash[83127]: audit 2024-09-13T00:53:58.430136+0000 mon.a (mon.0) 1522 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:53:59.325 INFO:teuthology.orchestra.run.smithi077.stdout:2156073582722 2024-09-13T00:53:59.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:59 smithi088 bash[68960]: audit 2024-09-13T00:53:58.309451+0000 mon.a (mon.0) 1521 : audit [DBG] from='client.? 172.21.15.77:0/27583279' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-13T00:53:59.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:53:59 smithi088 bash[68960]: audit 2024-09-13T00:53:58.430136+0000 mon.a (mon.0) 1522 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:53:59.458 INFO:teuthology.orchestra.run.smithi077.stdout:2787433775193 2024-09-13T00:54:00.047 INFO:teuthology.orchestra.run.smithi077.stdout:2701534429285 2024-09-13T00:54:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:00 smithi077 bash[83127]: cluster 2024-09-13T00:53:58.897206+0000 mgr.y (mgr.25304) 543 : cluster [DBG] pgmap v1119: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:00 smithi077 bash[83127]: audit 2024-09-13T00:53:59.328275+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.77:0/288520969' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T00:54:00.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:00 smithi077 bash[83127]: audit 2024-09-13T00:53:59.461507+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.77:0/3131494769' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T00:54:00.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:00 smithi077 bash[80681]: cluster 2024-09-13T00:53:58.897206+0000 mgr.y (mgr.25304) 543 : cluster [DBG] pgmap v1119: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:00.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:00 smithi077 bash[80681]: audit 2024-09-13T00:53:59.328275+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.77:0/288520969' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T00:54:00.280 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:00 smithi077 bash[80681]: audit 2024-09-13T00:53:59.461507+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.77:0/3131494769' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T00:54:00.332 INFO:teuthology.orchestra.run.smithi077.stdout:154618822985 2024-09-13T00:54:00.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:00 smithi088 bash[68960]: cluster 2024-09-13T00:53:58.897206+0000 mgr.y (mgr.25304) 543 : cluster [DBG] pgmap v1119: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:00.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:00 smithi088 bash[68960]: audit 2024-09-13T00:53:59.328275+0000 mon.c (mon.2) 49 : audit [DBG] from='client.? 172.21.15.77:0/288520969' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-13T00:54:00.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:00 smithi088 bash[68960]: audit 2024-09-13T00:53:59.461507+0000 mon.c (mon.2) 50 : audit [DBG] from='client.? 172.21.15.77:0/3131494769' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-13T00:54:01.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:01 smithi077 bash[83127]: audit 2024-09-13T00:54:00.050532+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.77:0/863364371' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T00:54:01.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:01 smithi077 bash[83127]: audit 2024-09-13T00:54:00.334990+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.77:0/3385932904' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T00:54:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:01 smithi077 bash[80681]: audit 2024-09-13T00:54:00.050532+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.77:0/863364371' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T00:54:01.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:01 smithi077 bash[80681]: audit 2024-09-13T00:54:00.334990+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.77:0/3385932904' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T00:54:01.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:01 smithi088 bash[68960]: audit 2024-09-13T00:54:00.050532+0000 mon.c (mon.2) 51 : audit [DBG] from='client.? 172.21.15.77:0/863364371' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-13T00:54:01.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:01 smithi088 bash[68960]: audit 2024-09-13T00:54:00.334990+0000 mon.c (mon.2) 52 : audit [DBG] from='client.? 172.21.15.77:0/3385932904' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-13T00:54:01.405 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2314987372661 got 2314987372666 for osd.1 2024-09-13T00:54:01.405 DEBUG:teuthology.parallel:result is None 2024-09-13T00:54:01.561 INFO:teuthology.orchestra.run.smithi077.stdout:2761663971423 2024-09-13T00:54:01.786 INFO:teuthology.orchestra.run.smithi077.stdout:206158430518 2024-09-13T00:54:02.232 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:02 smithi077 bash[83127]: cluster 2024-09-13T00:54:00.898124+0000 mgr.y (mgr.25304) 544 : cluster [DBG] pgmap v1120: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:02.232 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:02 smithi077 bash[83127]: audit 2024-09-13T00:54:01.563793+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.77:0/2344635133' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T00:54:02.232 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:02 smithi077 bash[83127]: audit 2024-09-13T00:54:01.789656+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.77:0/2100080526' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T00:54:02.232 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:02 smithi077 bash[80681]: cluster 2024-09-13T00:54:00.898124+0000 mgr.y (mgr.25304) 544 : cluster [DBG] pgmap v1120: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:02.232 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:02 smithi077 bash[80681]: audit 2024-09-13T00:54:01.563793+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.77:0/2344635133' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T00:54:02.232 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:02 smithi077 bash[80681]: audit 2024-09-13T00:54:01.789656+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.77:0/2100080526' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T00:54:02.314 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:02 smithi088 bash[68960]: cluster 2024-09-13T00:54:00.898124+0000 mgr.y (mgr.25304) 544 : cluster [DBG] pgmap v1120: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:02.314 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:02 smithi088 bash[68960]: audit 2024-09-13T00:54:01.563793+0000 mon.c (mon.2) 53 : audit [DBG] from='client.? 172.21.15.77:0/2344635133' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-13T00:54:02.315 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:02 smithi088 bash[68960]: audit 2024-09-13T00:54:01.789656+0000 mon.c (mon.2) 54 : audit [DBG] from='client.? 172.21.15.77:0/2100080526' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-13T00:54:02.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:54:02 smithi088 bash[61791]: ts=2024-09-13T00:54:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:54:03.215 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2156073582716 got 2156073582722 for osd.0 2024-09-13T00:54:03.215 DEBUG:teuthology.parallel:result is None 2024-09-13T00:54:03.348 INFO:teuthology.orchestra.run.smithi077.stdout:180388626752 2024-09-13T00:54:04.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:04 smithi077 bash[80681]: cluster 2024-09-13T00:54:02.898930+0000 mgr.y (mgr.25304) 545 : cluster [DBG] pgmap v1121: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:04.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:04 smithi077 bash[80681]: audit 2024-09-13T00:54:03.351437+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.77:0/2793582862' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T00:54:04.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:04 smithi077 bash[83127]: cluster 2024-09-13T00:54:02.898930+0000 mgr.y (mgr.25304) 545 : cluster [DBG] pgmap v1121: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:04.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:04 smithi077 bash[83127]: audit 2024-09-13T00:54:03.351437+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.77:0/2793582862' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T00:54:04.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:04 smithi088 bash[68960]: cluster 2024-09-13T00:54:02.898930+0000 mgr.y (mgr.25304) 545 : cluster [DBG] pgmap v1121: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:04.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:04 smithi088 bash[68960]: audit 2024-09-13T00:54:03.351437+0000 mon.c (mon.2) 55 : audit [DBG] from='client.? 172.21.15.77:0/2793582862' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-13T00:54:04.711 INFO:tasks.cephadm.ceph_manager.ceph:need seq 154618822980 got 154618822985 for osd.5 2024-09-13T00:54:04.711 DEBUG:teuthology.parallel:result is None 2024-09-13T00:54:04.833 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2787433775187 got 2787433775193 for osd.4 2024-09-13T00:54:04.833 DEBUG:teuthology.parallel:result is None 2024-09-13T00:54:04.909 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2701534429280 got 2701534429285 for osd.2 2024-09-13T00:54:04.909 DEBUG:teuthology.parallel:result is None 2024-09-13T00:54:06.073 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2761663971418 got 2761663971423 for osd.3 2024-09-13T00:54:06.073 DEBUG:teuthology.parallel:result is None 2024-09-13T00:54:06.270 INFO:tasks.cephadm.ceph_manager.ceph:need seq 206158430513 got 206158430518 for osd.7 2024-09-13T00:54:06.271 DEBUG:teuthology.parallel:result is None 2024-09-13T00:54:06.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:06 smithi088 bash[68960]: cluster 2024-09-13T00:54:04.899635+0000 mgr.y (mgr.25304) 546 : cluster [DBG] pgmap v1122: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:06.363 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:06 smithi077 bash[80681]: cluster 2024-09-13T00:54:04.899635+0000 mgr.y (mgr.25304) 546 : cluster [DBG] pgmap v1122: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:06.364 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:06 smithi077 bash[83127]: cluster 2024-09-13T00:54:04.899635+0000 mgr.y (mgr.25304) 546 : cluster [DBG] pgmap v1122: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:06.364 INFO:tasks.cephadm.ceph_manager.ceph:need seq 180388626747 got 180388626752 for osd.6 2024-09-13T00:54:06.365 DEBUG:teuthology.parallel:result is None 2024-09-13T00:54:06.365 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-13T00:54:06.365 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T00:54:08.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:08 smithi088 bash[68960]: cluster 2024-09-13T00:54:06.900343+0000 mgr.y (mgr.25304) 547 : cluster [DBG] pgmap v1123: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:08.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:08 smithi077 bash[80681]: cluster 2024-09-13T00:54:06.900343+0000 mgr.y (mgr.25304) 547 : cluster [DBG] pgmap v1123: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:08.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:08 smithi077 bash[83127]: cluster 2024-09-13T00:54:06.900343+0000 mgr.y (mgr.25304) 547 : cluster [DBG] pgmap v1123: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:08.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:54:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:54:08] "GET /metrics HTTP/1.1" 200 34265 "" "Prometheus/2.43.0" 2024-09-13T00:54:10.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:10 smithi088 bash[68960]: cluster 2024-09-13T00:54:08.901163+0000 mgr.y (mgr.25304) 548 : cluster [DBG] pgmap v1124: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:10.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:10 smithi077 bash[80681]: cluster 2024-09-13T00:54:08.901163+0000 mgr.y (mgr.25304) 548 : cluster [DBG] pgmap v1124: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:10 smithi077 bash[83127]: cluster 2024-09-13T00:54:08.901163+0000 mgr.y (mgr.25304) 548 : cluster [DBG] pgmap v1124: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:11.150 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:54:12.315 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:12 smithi088 bash[68960]: cluster 2024-09-13T00:54:10.902102+0000 mgr.y (mgr.25304) 549 : cluster [DBG] pgmap v1125: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:12.417 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:12 smithi077 bash[80681]: cluster 2024-09-13T00:54:10.902102+0000 mgr.y (mgr.25304) 549 : cluster [DBG] pgmap v1125: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:12.417 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:12 smithi077 bash[83127]: cluster 2024-09-13T00:54:10.902102+0000 mgr.y (mgr.25304) 549 : cluster [DBG] pgmap v1125: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:12.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:54:12 smithi088 bash[61791]: ts=2024-09-13T00:54:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:54:14.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:14 smithi088 bash[68960]: cluster 2024-09-13T00:54:12.902879+0000 mgr.y (mgr.25304) 550 : cluster [DBG] pgmap v1126: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:14.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:14 smithi088 bash[68960]: audit 2024-09-13T00:54:13.431173+0000 mon.a (mon.0) 1523 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:54:14.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:14 smithi077 bash[80681]: cluster 2024-09-13T00:54:12.902879+0000 mgr.y (mgr.25304) 550 : cluster [DBG] pgmap v1126: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:14 smithi077 bash[80681]: audit 2024-09-13T00:54:13.431173+0000 mon.a (mon.0) 1523 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:54:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:14 smithi077 bash[83127]: cluster 2024-09-13T00:54:12.902879+0000 mgr.y (mgr.25304) 550 : cluster [DBG] pgmap v1126: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:14 smithi077 bash[83127]: audit 2024-09-13T00:54:13.431173+0000 mon.a (mon.0) 1523 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:54:14.895 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:54:14.897 INFO:teuthology.orchestra.run.smithi077.stderr:dumped all 2024-09-13T00:54:15.727 INFO:teuthology.orchestra.run.smithi077.stdout:{"pg_ready":true,"pg_map":{"version":1126,"stamp":"2024-09-13T00:54:12.902410+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595923,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":452,"num_read_kb":889,"num_write":237,"num_write_kb":1660,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":177,"ondisk_log_size":177,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1454196,"kb_used_data":125500,"kb_used_omap":327,"kb_used_meta":1178168,"kb_avail":748507020,"statfs":{"total":767960285184,"available":766471188480,"internally_reserved":0,"allocated":128512000,"data_stored":81080198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":335765,"internal_metadata":1206444139},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004887"},"pg_stats":[{"pgid":"3.1f","version":"63'5","reported_seq":911,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.327228+0000","last_change":"2024-09-13T00:45:41.626626+0000","last_active":"2024-09-13T00:52:40.327228+0000","last_peered":"2024-09-13T00:52:40.327228+0000","last_clean":"2024-09-13T00:52:40.327228+0000","last_became_active":"2024-09-13T00:45:41.626278+0000","last_became_peered":"2024-09-13T00:45:41.626278+0000","last_unstale":"2024-09-13T00:52:40.327228+0000","last_undegraded":"2024-09-13T00:52:40.327228+0000","last_fullsized":"2024-09-13T00:52:40.327228+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:42:58.455220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":923,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.258186+0000","last_change":"2024-09-13T00:46:45.517960+0000","last_active":"2024-09-13T00:52:40.258186+0000","last_peered":"2024-09-13T00:52:40.258186+0000","last_clean":"2024-09-13T00:52:40.258186+0000","last_became_active":"2024-09-13T00:46:45.517434+0000","last_became_peered":"2024-09-13T00:46:45.517434+0000","last_unstale":"2024-09-13T00:52:40.258186+0000","last_undegraded":"2024-09-13T00:52:40.258186+0000","last_fullsized":"2024-09-13T00:52:40.258186+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:40:06.681450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"63'2","reported_seq":919,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.265412+0000","last_change":"2024-09-13T00:46:15.330699+0000","last_active":"2024-09-13T00:52:39.265412+0000","last_peered":"2024-09-13T00:52:39.265412+0000","last_clean":"2024-09-13T00:52:39.265412+0000","last_became_active":"2024-09-13T00:46:15.325486+0000","last_became_peered":"2024-09-13T00:46:15.325486+0000","last_unstale":"2024-09-13T00:52:39.265412+0000","last_undegraded":"2024-09-13T00:52:39.265412+0000","last_fullsized":"2024-09-13T00:52:39.265412+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:13:40.129831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":940,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.321114+0000","last_change":"2024-09-13T00:46:15.321514+0000","last_active":"2024-09-13T00:52:40.321114+0000","last_peered":"2024-09-13T00:52:40.321114+0000","last_clean":"2024-09-13T00:52:40.321114+0000","last_became_active":"2024-09-13T00:46:15.320037+0000","last_became_peered":"2024-09-13T00:46:15.320037+0000","last_unstale":"2024-09-13T00:52:40.321114+0000","last_undegraded":"2024-09-13T00:52:40.321114+0000","last_fullsized":"2024-09-13T00:52:40.321114+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:40:56.856055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"62'2","reported_seq":913,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.259414+0000","last_change":"2024-09-13T00:46:45.507901+0000","last_active":"2024-09-13T00:52:39.259414+0000","last_peered":"2024-09-13T00:52:39.259414+0000","last_clean":"2024-09-13T00:52:39.259414+0000","last_became_active":"2024-09-13T00:46:45.507081+0000","last_became_peered":"2024-09-13T00:46:45.507081+0000","last_unstale":"2024-09-13T00:52:39.259414+0000","last_undegraded":"2024-09-13T00:52:39.259414+0000","last_fullsized":"2024-09-13T00:52:39.259414+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:12:06.783019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":943,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.260620+0000","last_change":"2024-09-13T00:46:45.509213+0000","last_active":"2024-09-13T00:52:39.260620+0000","last_peered":"2024-09-13T00:52:39.260620+0000","last_clean":"2024-09-13T00:52:39.260620+0000","last_became_active":"2024-09-13T00:46:45.508648+0000","last_became_peered":"2024-09-13T00:46:45.508648+0000","last_unstale":"2024-09-13T00:52:39.260620+0000","last_undegraded":"2024-09-13T00:52:39.260620+0000","last_fullsized":"2024-09-13T00:52:39.260620+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:21:41.245775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"62'2","reported_seq":930,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.258592+0000","last_change":"2024-09-13T00:46:45.522136+0000","last_active":"2024-09-13T00:52:39.258592+0000","last_peered":"2024-09-13T00:52:39.258592+0000","last_clean":"2024-09-13T00:52:39.258592+0000","last_became_active":"2024-09-13T00:46:45.521754+0000","last_became_peered":"2024-09-13T00:46:45.521754+0000","last_unstale":"2024-09-13T00:52:39.258592+0000","last_undegraded":"2024-09-13T00:52:39.258592+0000","last_fullsized":"2024-09-13T00:52:39.258592+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:31:59.251880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":943,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.261353+0000","last_change":"2024-09-13T00:46:45.535236+0000","last_active":"2024-09-13T00:52:39.261353+0000","last_peered":"2024-09-13T00:52:39.261353+0000","last_clean":"2024-09-13T00:52:39.261353+0000","last_became_active":"2024-09-13T00:46:45.534956+0000","last_became_peered":"2024-09-13T00:46:45.534956+0000","last_unstale":"2024-09-13T00:52:39.261353+0000","last_undegraded":"2024-09-13T00:52:39.261353+0000","last_fullsized":"2024-09-13T00:52:39.261353+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:55:47.169063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"63'2","reported_seq":905,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.323022+0000","last_change":"2024-09-13T00:46:45.518960+0000","last_active":"2024-09-13T00:52:40.323022+0000","last_peered":"2024-09-13T00:52:40.323022+0000","last_clean":"2024-09-13T00:52:40.323022+0000","last_became_active":"2024-09-13T00:46:45.517233+0000","last_became_peered":"2024-09-13T00:46:45.517233+0000","last_unstale":"2024-09-13T00:52:40.323022+0000","last_undegraded":"2024-09-13T00:52:40.323022+0000","last_fullsized":"2024-09-13T00:52:40.323022+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:33:04.004456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":941,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.259178+0000","last_change":"2024-09-13T00:46:15.344315+0000","last_active":"2024-09-13T00:52:40.259178+0000","last_peered":"2024-09-13T00:52:40.259178+0000","last_clean":"2024-09-13T00:52:40.259178+0000","last_became_active":"2024-09-13T00:46:15.343998+0000","last_became_peered":"2024-09-13T00:46:15.343998+0000","last_unstale":"2024-09-13T00:52:40.259178+0000","last_undegraded":"2024-09-13T00:52:40.259178+0000","last_fullsized":"2024-09-13T00:52:40.259178+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T12:20:50.373554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"63'2","reported_seq":932,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.261036+0000","last_change":"2024-09-13T00:46:45.529526+0000","last_active":"2024-09-13T00:52:39.261036+0000","last_peered":"2024-09-13T00:52:39.261036+0000","last_clean":"2024-09-13T00:52:39.261036+0000","last_became_active":"2024-09-13T00:46:45.527232+0000","last_became_peered":"2024-09-13T00:46:45.527232+0000","last_unstale":"2024-09-13T00:52:39.261036+0000","last_undegraded":"2024-09-13T00:52:39.261036+0000","last_fullsized":"2024-09-13T00:52:39.261036+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:44:35.336089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":938,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.319032+0000","last_change":"2024-09-13T00:46:45.522928+0000","last_active":"2024-09-13T00:52:40.319032+0000","last_peered":"2024-09-13T00:52:40.319032+0000","last_clean":"2024-09-13T00:52:40.319032+0000","last_became_active":"2024-09-13T00:46:45.521955+0000","last_became_peered":"2024-09-13T00:46:45.521955+0000","last_unstale":"2024-09-13T00:52:40.319032+0000","last_undegraded":"2024-09-13T00:52:40.319032+0000","last_fullsized":"2024-09-13T00:52:40.319032+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:01:30.288065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":918,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.324428+0000","last_change":"2024-09-13T00:46:45.530887+0000","last_active":"2024-09-13T00:52:40.324428+0000","last_peered":"2024-09-13T00:52:40.324428+0000","last_clean":"2024-09-13T00:52:40.324428+0000","last_became_active":"2024-09-13T00:46:45.530400+0000","last_became_peered":"2024-09-13T00:46:45.530400+0000","last_unstale":"2024-09-13T00:52:40.324428+0000","last_undegraded":"2024-09-13T00:52:40.324428+0000","last_fullsized":"2024-09-13T00:52:40.324428+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:31:55.473617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":901,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.319151+0000","last_change":"2024-09-13T00:46:15.331478+0000","last_active":"2024-09-13T00:52:40.319151+0000","last_peered":"2024-09-13T00:52:40.319151+0000","last_clean":"2024-09-13T00:52:40.319151+0000","last_became_active":"2024-09-13T00:46:15.331202+0000","last_became_peered":"2024-09-13T00:46:15.331202+0000","last_unstale":"2024-09-13T00:52:40.319151+0000","last_undegraded":"2024-09-13T00:52:40.319151+0000","last_fullsized":"2024-09-13T00:52:40.319151+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:35:28.914097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":941,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.313324+0000","last_change":"2024-09-13T00:46:15.341826+0000","last_active":"2024-09-13T00:52:40.313324+0000","last_peered":"2024-09-13T00:52:40.313324+0000","last_clean":"2024-09-13T00:52:40.313324+0000","last_became_active":"2024-09-13T00:46:15.341132+0000","last_became_peered":"2024-09-13T00:46:15.341132+0000","last_unstale":"2024-09-13T00:52:40.313324+0000","last_undegraded":"2024-09-13T00:52:40.313324+0000","last_fullsized":"2024-09-13T00:52:40.313324+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:37:09.570985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":915,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.432015+0000","last_change":"2024-09-13T00:43:56.096526+0000","last_active":"2024-09-13T00:52:39.432015+0000","last_peered":"2024-09-13T00:52:39.432015+0000","last_clean":"2024-09-13T00:52:39.432015+0000","last_became_active":"2024-09-13T00:43:56.094929+0000","last_became_peered":"2024-09-13T00:43:56.094929+0000","last_unstale":"2024-09-13T00:52:39.432015+0000","last_undegraded":"2024-09-13T00:52:39.432015+0000","last_fullsized":"2024-09-13T00:52:39.432015+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:24:56.143144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":923,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.313961+0000","last_change":"2024-09-13T00:46:15.339591+0000","last_active":"2024-09-13T00:52:40.313961+0000","last_peered":"2024-09-13T00:52:40.313961+0000","last_clean":"2024-09-13T00:52:40.313961+0000","last_became_active":"2024-09-13T00:46:15.338782+0000","last_became_peered":"2024-09-13T00:46:15.338782+0000","last_unstale":"2024-09-13T00:52:40.313961+0000","last_undegraded":"2024-09-13T00:52:40.313961+0000","last_fullsized":"2024-09-13T00:52:40.313961+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:11:30.414422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":900,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.326970+0000","last_change":"2024-09-13T00:46:15.328788+0000","last_active":"2024-09-13T00:52:40.326970+0000","last_peered":"2024-09-13T00:52:40.326970+0000","last_clean":"2024-09-13T00:52:40.326970+0000","last_became_active":"2024-09-13T00:46:15.328123+0000","last_became_peered":"2024-09-13T00:46:15.328123+0000","last_unstale":"2024-09-13T00:52:40.326970+0000","last_undegraded":"2024-09-13T00:52:40.326970+0000","last_fullsized":"2024-09-13T00:52:40.326970+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:47:38.824723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":906,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.324737+0000","last_change":"2024-09-13T00:46:15.333000+0000","last_active":"2024-09-13T00:52:40.324737+0000","last_peered":"2024-09-13T00:52:40.324737+0000","last_clean":"2024-09-13T00:52:40.324737+0000","last_became_active":"2024-09-13T00:46:15.332581+0000","last_became_peered":"2024-09-13T00:46:15.332581+0000","last_unstale":"2024-09-13T00:52:40.324737+0000","last_undegraded":"2024-09-13T00:52:40.324737+0000","last_fullsized":"2024-09-13T00:52:40.324737+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:01:06.004637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":910,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.259738+0000","last_change":"2024-09-13T00:43:56.097452+0000","last_active":"2024-09-13T00:52:39.259738+0000","last_peered":"2024-09-13T00:52:39.259738+0000","last_clean":"2024-09-13T00:52:39.259738+0000","last_became_active":"2024-09-13T00:43:56.096578+0000","last_became_peered":"2024-09-13T00:43:56.096578+0000","last_unstale":"2024-09-13T00:52:39.259738+0000","last_undegraded":"2024-09-13T00:52:39.259738+0000","last_fullsized":"2024-09-13T00:52:39.259738+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:59:21.650017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"62'3","reported_seq":935,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.279968+0000","last_change":"2024-09-13T00:46:45.503242+0000","last_active":"2024-09-13T00:52:39.279968+0000","last_peered":"2024-09-13T00:52:39.279968+0000","last_clean":"2024-09-13T00:52:39.279968+0000","last_became_active":"2024-09-13T00:46:45.502312+0000","last_became_peered":"2024-09-13T00:46:45.502312+0000","last_unstale":"2024-09-13T00:52:39.279968+0000","last_undegraded":"2024-09-13T00:52:39.279968+0000","last_fullsized":"2024-09-13T00:52:39.279968+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:20:37.698979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":903,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.317441+0000","last_change":"2024-09-13T00:45:41.598285+0000","last_active":"2024-09-13T00:52:40.317441+0000","last_peered":"2024-09-13T00:52:40.317441+0000","last_clean":"2024-09-13T00:52:40.317441+0000","last_became_active":"2024-09-13T00:45:41.597845+0000","last_became_peered":"2024-09-13T00:45:41.597845+0000","last_unstale":"2024-09-13T00:52:40.317441+0000","last_undegraded":"2024-09-13T00:52:40.317441+0000","last_fullsized":"2024-09-13T00:52:40.317441+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:58:24.168311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"368'24","reported_seq":911,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.260827+0000","last_change":"2024-09-13T00:46:45.493995+0000","last_active":"2024-09-13T00:52:40.260827+0000","last_peered":"2024-09-13T00:52:40.260827+0000","last_clean":"2024-09-13T00:52:40.260827+0000","last_became_active":"2024-09-13T00:46:45.492736+0000","last_became_peered":"2024-09-13T00:46:45.492736+0000","last_unstale":"2024-09-13T00:52:40.260827+0000","last_undegraded":"2024-09-13T00:52:40.260827+0000","last_fullsized":"2024-09-13T00:52:40.260827+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":24,"log_dups_size":0,"ondisk_log_size":24,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:12:06.114096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4750,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":906,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.260843+0000","last_change":"2024-09-13T00:46:45.529941+0000","last_active":"2024-09-13T00:52:40.260843+0000","last_peered":"2024-09-13T00:52:40.260843+0000","last_clean":"2024-09-13T00:52:40.260843+0000","last_became_active":"2024-09-13T00:46:45.529071+0000","last_became_peered":"2024-09-13T00:46:45.529071+0000","last_unstale":"2024-09-13T00:52:40.260843+0000","last_undegraded":"2024-09-13T00:52:40.260843+0000","last_fullsized":"2024-09-13T00:52:40.260843+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:54:30.695569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":902,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.262323+0000","last_change":"2024-09-13T00:46:45.515148+0000","last_active":"2024-09-13T00:52:40.262323+0000","last_peered":"2024-09-13T00:52:40.262323+0000","last_clean":"2024-09-13T00:52:40.262323+0000","last_became_active":"2024-09-13T00:46:45.500689+0000","last_became_peered":"2024-09-13T00:46:45.500689+0000","last_unstale":"2024-09-13T00:52:40.262323+0000","last_undegraded":"2024-09-13T00:52:40.262323+0000","last_fullsized":"2024-09-13T00:52:40.262323+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:22:50.405443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":928,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.432161+0000","last_change":"2024-09-13T00:45:41.629188+0000","last_active":"2024-09-13T00:52:39.432161+0000","last_peered":"2024-09-13T00:52:39.432161+0000","last_clean":"2024-09-13T00:52:39.432161+0000","last_became_active":"2024-09-13T00:45:41.626966+0000","last_became_peered":"2024-09-13T00:45:41.626966+0000","last_unstale":"2024-09-13T00:52:39.432161+0000","last_undegraded":"2024-09-13T00:52:39.432161+0000","last_fullsized":"2024-09-13T00:52:39.432161+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:06:08.792854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00135603,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":920,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.318851+0000","last_change":"2024-09-13T00:46:45.519936+0000","last_active":"2024-09-13T00:52:40.318851+0000","last_peered":"2024-09-13T00:52:40.318851+0000","last_clean":"2024-09-13T00:52:40.318851+0000","last_became_active":"2024-09-13T00:46:45.519473+0000","last_became_peered":"2024-09-13T00:46:45.519473+0000","last_unstale":"2024-09-13T00:52:40.318851+0000","last_undegraded":"2024-09-13T00:52:40.318851+0000","last_fullsized":"2024-09-13T00:52:40.318851+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:26:37.351469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":923,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.264234+0000","last_change":"2024-09-13T00:46:15.334557+0000","last_active":"2024-09-13T00:52:39.264234+0000","last_peered":"2024-09-13T00:52:39.264234+0000","last_clean":"2024-09-13T00:52:39.264234+0000","last_became_active":"2024-09-13T00:46:15.333534+0000","last_became_peered":"2024-09-13T00:46:15.333534+0000","last_unstale":"2024-09-13T00:52:39.264234+0000","last_undegraded":"2024-09-13T00:52:39.264234+0000","last_fullsized":"2024-09-13T00:52:39.264234+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:39:10.101248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":913,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.272636+0000","last_change":"2024-09-13T00:46:45.529937+0000","last_active":"2024-09-13T00:52:39.272636+0000","last_peered":"2024-09-13T00:52:39.272636+0000","last_clean":"2024-09-13T00:52:39.272636+0000","last_became_active":"2024-09-13T00:46:45.529259+0000","last_became_peered":"2024-09-13T00:46:45.529259+0000","last_unstale":"2024-09-13T00:52:39.272636+0000","last_undegraded":"2024-09-13T00:52:39.272636+0000","last_fullsized":"2024-09-13T00:52:39.272636+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:56:18.590692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"62'2","reported_seq":927,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.320734+0000","last_change":"2024-09-13T00:46:15.335239+0000","last_active":"2024-09-13T00:52:40.320734+0000","last_peered":"2024-09-13T00:52:40.320734+0000","last_clean":"2024-09-13T00:52:40.320734+0000","last_became_active":"2024-09-13T00:46:15.334885+0000","last_became_peered":"2024-09-13T00:46:15.334885+0000","last_unstale":"2024-09-13T00:52:40.320734+0000","last_undegraded":"2024-09-13T00:52:40.320734+0000","last_fullsized":"2024-09-13T00:52:40.320734+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:43:15.769703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":886,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.323382+0000","last_change":"2024-09-13T00:43:18.180239+0000","last_active":"2024-09-13T00:52:40.323382+0000","last_peered":"2024-09-13T00:52:40.323382+0000","last_clean":"2024-09-13T00:52:40.323382+0000","last_became_active":"2024-09-13T00:43:18.179413+0000","last_became_peered":"2024-09-13T00:43:18.179413+0000","last_unstale":"2024-09-13T00:52:40.323382+0000","last_undegraded":"2024-09-13T00:52:40.323382+0000","last_fullsized":"2024-09-13T00:52:40.323382+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:18:03.116471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":942,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.280376+0000","last_change":"2024-09-13T00:46:45.503256+0000","last_active":"2024-09-13T00:52:39.280376+0000","last_peered":"2024-09-13T00:52:39.280376+0000","last_clean":"2024-09-13T00:52:39.280376+0000","last_became_active":"2024-09-13T00:46:45.502770+0000","last_became_peered":"2024-09-13T00:46:45.502770+0000","last_unstale":"2024-09-13T00:52:39.280376+0000","last_undegraded":"2024-09-13T00:52:39.280376+0000","last_fullsized":"2024-09-13T00:52:39.280376+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:07:22.957742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077602300000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":903,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.272559+0000","last_change":"2024-09-13T00:45:41.601473+0000","last_active":"2024-09-13T00:52:39.272559+0000","last_peered":"2024-09-13T00:52:39.272559+0000","last_clean":"2024-09-13T00:52:39.272559+0000","last_became_active":"2024-09-13T00:45:41.600376+0000","last_became_peered":"2024-09-13T00:45:41.600376+0000","last_unstale":"2024-09-13T00:52:39.272559+0000","last_undegraded":"2024-09-13T00:52:39.272559+0000","last_fullsized":"2024-09-13T00:52:39.272559+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:37:33.287548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":903,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.271915+0000","last_change":"2024-09-13T00:46:45.532330+0000","last_active":"2024-09-13T00:52:39.271915+0000","last_peered":"2024-09-13T00:52:39.271915+0000","last_clean":"2024-09-13T00:52:39.271915+0000","last_became_active":"2024-09-13T00:46:45.531246+0000","last_became_peered":"2024-09-13T00:46:45.531246+0000","last_unstale":"2024-09-13T00:52:39.271915+0000","last_undegraded":"2024-09-13T00:52:39.271915+0000","last_fullsized":"2024-09-13T00:52:39.271915+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:49:33.378939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":908,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.272295+0000","last_change":"2024-09-13T00:43:56.101142+0000","last_active":"2024-09-13T00:52:39.272295+0000","last_peered":"2024-09-13T00:52:39.272295+0000","last_clean":"2024-09-13T00:52:39.272295+0000","last_became_active":"2024-09-13T00:43:56.098893+0000","last_became_peered":"2024-09-13T00:43:56.098893+0000","last_unstale":"2024-09-13T00:52:39.272295+0000","last_undegraded":"2024-09-13T00:52:39.272295+0000","last_fullsized":"2024-09-13T00:52:39.272295+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:04:44.962433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053219400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":927,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.259613+0000","last_change":"2024-09-13T00:46:15.327798+0000","last_active":"2024-09-13T00:52:39.259613+0000","last_peered":"2024-09-13T00:52:39.259613+0000","last_clean":"2024-09-13T00:52:39.259613+0000","last_became_active":"2024-09-13T00:46:15.327504+0000","last_became_peered":"2024-09-13T00:46:15.327504+0000","last_unstale":"2024-09-13T00:52:39.259613+0000","last_undegraded":"2024-09-13T00:52:39.259613+0000","last_fullsized":"2024-09-13T00:52:39.259613+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:02:29.779493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":918,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.272772+0000","last_change":"2024-09-13T00:46:15.354292+0000","last_active":"2024-09-13T00:52:39.272772+0000","last_peered":"2024-09-13T00:52:39.272772+0000","last_clean":"2024-09-13T00:52:39.272772+0000","last_became_active":"2024-09-13T00:46:15.352070+0000","last_became_peered":"2024-09-13T00:46:15.352070+0000","last_unstale":"2024-09-13T00:52:39.272772+0000","last_undegraded":"2024-09-13T00:52:39.272772+0000","last_fullsized":"2024-09-13T00:52:39.272772+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:14:17.847476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":911,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.260725+0000","last_change":"2024-09-13T00:46:45.517569+0000","last_active":"2024-09-13T00:52:40.260725+0000","last_peered":"2024-09-13T00:52:40.260725+0000","last_clean":"2024-09-13T00:52:40.260725+0000","last_became_active":"2024-09-13T00:46:45.516270+0000","last_became_peered":"2024-09-13T00:46:45.516270+0000","last_unstale":"2024-09-13T00:52:40.260725+0000","last_undegraded":"2024-09-13T00:52:40.260725+0000","last_fullsized":"2024-09-13T00:52:40.260725+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:46:41.924008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"57'2","reported_seq":924,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.258313+0000","last_change":"2024-09-13T00:43:56.096241+0000","last_active":"2024-09-13T00:52:39.258313+0000","last_peered":"2024-09-13T00:52:39.258313+0000","last_clean":"2024-09-13T00:52:39.258313+0000","last_became_active":"2024-09-13T00:43:56.095239+0000","last_became_peered":"2024-09-13T00:43:56.095239+0000","last_unstale":"2024-09-13T00:52:39.258313+0000","last_undegraded":"2024-09-13T00:52:39.258313+0000","last_fullsized":"2024-09-13T00:52:39.258313+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:58:15.629030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002059707,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":895,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.272286+0000","last_change":"2024-09-13T00:43:56.114793+0000","last_active":"2024-09-13T00:52:39.272286+0000","last_peered":"2024-09-13T00:52:39.272286+0000","last_clean":"2024-09-13T00:52:39.272286+0000","last_became_active":"2024-09-13T00:43:56.109668+0000","last_became_peered":"2024-09-13T00:43:56.109668+0000","last_unstale":"2024-09-13T00:52:39.272286+0000","last_undegraded":"2024-09-13T00:52:39.272286+0000","last_fullsized":"2024-09-13T00:52:39.272286+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:32:06.789192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":904,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.322248+0000","last_change":"2024-09-13T00:45:41.597941+0000","last_active":"2024-09-13T00:52:40.322248+0000","last_peered":"2024-09-13T00:52:40.322248+0000","last_clean":"2024-09-13T00:52:40.322248+0000","last_became_active":"2024-09-13T00:45:41.597499+0000","last_became_peered":"2024-09-13T00:45:41.597499+0000","last_unstale":"2024-09-13T00:52:40.322248+0000","last_undegraded":"2024-09-13T00:52:40.322248+0000","last_fullsized":"2024-09-13T00:52:40.322248+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:56:08.198126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":919,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.258023+0000","last_change":"2024-09-13T00:46:15.312908+0000","last_active":"2024-09-13T00:52:40.258023+0000","last_peered":"2024-09-13T00:52:40.258023+0000","last_clean":"2024-09-13T00:52:40.258023+0000","last_became_active":"2024-09-13T00:46:15.312536+0000","last_became_peered":"2024-09-13T00:46:15.312536+0000","last_unstale":"2024-09-13T00:52:40.258023+0000","last_undegraded":"2024-09-13T00:52:40.258023+0000","last_fullsized":"2024-09-13T00:52:40.258023+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:57:23.320972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067555600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":916,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.319294+0000","last_change":"2024-09-13T00:45:41.619166+0000","last_active":"2024-09-13T00:52:40.319294+0000","last_peered":"2024-09-13T00:52:40.319294+0000","last_clean":"2024-09-13T00:52:40.319294+0000","last_became_active":"2024-09-13T00:45:41.618844+0000","last_became_peered":"2024-09-13T00:45:41.618844+0000","last_unstale":"2024-09-13T00:52:40.319294+0000","last_undegraded":"2024-09-13T00:52:40.319294+0000","last_fullsized":"2024-09-13T00:52:40.319294+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:02:14.691566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":916,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.330943+0000","last_change":"2024-09-13T00:46:45.496190+0000","last_active":"2024-09-13T00:52:40.330943+0000","last_peered":"2024-09-13T00:52:40.330943+0000","last_clean":"2024-09-13T00:52:40.330943+0000","last_became_active":"2024-09-13T00:46:45.495623+0000","last_became_peered":"2024-09-13T00:46:45.495623+0000","last_unstale":"2024-09-13T00:52:40.330943+0000","last_undegraded":"2024-09-13T00:52:40.330943+0000","last_fullsized":"2024-09-13T00:52:40.330943+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:33:20.680033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":942,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.280291+0000","last_change":"2024-09-13T00:43:56.094126+0000","last_active":"2024-09-13T00:52:39.280291+0000","last_peered":"2024-09-13T00:52:39.280291+0000","last_clean":"2024-09-13T00:52:39.280291+0000","last_became_active":"2024-09-13T00:43:56.093290+0000","last_became_peered":"2024-09-13T00:43:56.093290+0000","last_unstale":"2024-09-13T00:52:39.280291+0000","last_undegraded":"2024-09-13T00:52:39.280291+0000","last_fullsized":"2024-09-13T00:52:39.280291+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:26:40.546096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062469399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":915,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.432380+0000","last_change":"2024-09-13T00:46:45.486085+0000","last_active":"2024-09-13T00:52:39.432380+0000","last_peered":"2024-09-13T00:52:39.432380+0000","last_clean":"2024-09-13T00:52:39.432380+0000","last_became_active":"2024-09-13T00:46:45.485596+0000","last_became_peered":"2024-09-13T00:46:45.485596+0000","last_unstale":"2024-09-13T00:52:39.432380+0000","last_undegraded":"2024-09-13T00:52:39.432380+0000","last_fullsized":"2024-09-13T00:52:39.432380+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:54:51.909304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":905,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.264936+0000","last_change":"2024-09-13T00:46:15.337295+0000","last_active":"2024-09-13T00:52:39.264936+0000","last_peered":"2024-09-13T00:52:39.264936+0000","last_clean":"2024-09-13T00:52:39.264936+0000","last_became_active":"2024-09-13T00:46:15.336890+0000","last_became_peered":"2024-09-13T00:46:15.336890+0000","last_unstale":"2024-09-13T00:52:39.264936+0000","last_undegraded":"2024-09-13T00:52:39.264936+0000","last_fullsized":"2024-09-13T00:52:39.264936+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T12:31:43.952771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"55'1","reported_seq":923,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.258734+0000","last_change":"2024-09-13T00:45:41.620277+0000","last_active":"2024-09-13T00:52:39.258734+0000","last_peered":"2024-09-13T00:52:39.258734+0000","last_clean":"2024-09-13T00:52:39.258734+0000","last_became_active":"2024-09-13T00:45:41.619579+0000","last_became_peered":"2024-09-13T00:45:41.619579+0000","last_unstale":"2024-09-13T00:52:39.258734+0000","last_undegraded":"2024-09-13T00:52:39.258734+0000","last_fullsized":"2024-09-13T00:52:39.258734+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:10:40.119383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096918099999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"292'112","reported_seq":1072,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.279491+0000","last_change":"2024-09-13T00:43:18.168213+0000","last_active":"2024-09-13T00:52:39.279491+0000","last_peered":"2024-09-13T00:52:39.279491+0000","last_clean":"2024-09-13T00:52:39.279491+0000","last_became_active":"2024-09-13T00:43:18.167226+0000","last_became_peered":"2024-09-13T00:43:18.167226+0000","last_unstale":"2024-09-13T00:52:39.279491+0000","last_undegraded":"2024-09-13T00:52:39.279491+0000","last_fullsized":"2024-09-13T00:52:39.279491+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:24:16.220413+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:24:16.220413+0000","last_clean_scrub_stamp":"2024-09-13T00:24:16.220413+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:55:23.652528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":414,"num_read_kb":849,"num_write":173,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":918,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.321743+0000","last_change":"2024-09-13T00:46:45.530576+0000","last_active":"2024-09-13T00:52:40.321743+0000","last_peered":"2024-09-13T00:52:40.321743+0000","last_clean":"2024-09-13T00:52:40.321743+0000","last_became_active":"2024-09-13T00:46:45.529881+0000","last_became_peered":"2024-09-13T00:46:45.529881+0000","last_unstale":"2024-09-13T00:52:40.321743+0000","last_undegraded":"2024-09-13T00:52:40.321743+0000","last_fullsized":"2024-09-13T00:52:40.321743+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:52:55.873479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":924,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.313757+0000","last_change":"2024-09-13T00:46:15.341519+0000","last_active":"2024-09-13T00:52:40.313757+0000","last_peered":"2024-09-13T00:52:40.313757+0000","last_clean":"2024-09-13T00:52:40.313757+0000","last_became_active":"2024-09-13T00:46:15.340548+0000","last_became_peered":"2024-09-13T00:46:15.340548+0000","last_unstale":"2024-09-13T00:52:40.313757+0000","last_undegraded":"2024-09-13T00:52:40.313757+0000","last_fullsized":"2024-09-13T00:52:40.313757+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:33:57.368092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":906,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.262146+0000","last_change":"2024-09-13T00:46:45.526955+0000","last_active":"2024-09-13T00:52:40.262146+0000","last_peered":"2024-09-13T00:52:40.262146+0000","last_clean":"2024-09-13T00:52:40.262146+0000","last_became_active":"2024-09-13T00:46:45.526699+0000","last_became_peered":"2024-09-13T00:46:45.526699+0000","last_unstale":"2024-09-13T00:52:40.262146+0000","last_undegraded":"2024-09-13T00:52:40.262146+0000","last_fullsized":"2024-09-13T00:52:40.262146+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:39:27.703444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":914,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.260160+0000","last_change":"2024-09-13T00:46:45.521123+0000","last_active":"2024-09-13T00:52:40.260160+0000","last_peered":"2024-09-13T00:52:40.260160+0000","last_clean":"2024-09-13T00:52:40.260160+0000","last_became_active":"2024-09-13T00:46:45.519894+0000","last_became_peered":"2024-09-13T00:46:45.519894+0000","last_unstale":"2024-09-13T00:52:40.260160+0000","last_undegraded":"2024-09-13T00:52:40.260160+0000","last_fullsized":"2024-09-13T00:52:40.260160+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:27:03.451547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":930,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.261925+0000","last_change":"2024-09-13T00:46:45.521910+0000","last_active":"2024-09-13T00:52:40.261925+0000","last_peered":"2024-09-13T00:52:40.261925+0000","last_clean":"2024-09-13T00:52:40.261925+0000","last_became_active":"2024-09-13T00:46:45.520547+0000","last_became_peered":"2024-09-13T00:46:45.520547+0000","last_unstale":"2024-09-13T00:52:40.261925+0000","last_undegraded":"2024-09-13T00:52:40.261925+0000","last_fullsized":"2024-09-13T00:52:40.261925+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:57:43.072817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":933,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.431407+0000","last_change":"2024-09-13T00:46:45.518980+0000","last_active":"2024-09-13T00:52:39.431407+0000","last_peered":"2024-09-13T00:52:39.431407+0000","last_clean":"2024-09-13T00:52:39.431407+0000","last_became_active":"2024-09-13T00:46:45.518676+0000","last_became_peered":"2024-09-13T00:46:45.518676+0000","last_unstale":"2024-09-13T00:52:39.431407+0000","last_undegraded":"2024-09-13T00:52:39.431407+0000","last_fullsized":"2024-09-13T00:52:39.431407+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:07:29.535519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":927,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.261060+0000","last_change":"2024-09-13T00:46:45.509062+0000","last_active":"2024-09-13T00:52:40.261060+0000","last_peered":"2024-09-13T00:52:40.261060+0000","last_clean":"2024-09-13T00:52:40.261060+0000","last_became_active":"2024-09-13T00:46:45.504287+0000","last_became_peered":"2024-09-13T00:46:45.504287+0000","last_unstale":"2024-09-13T00:52:40.261060+0000","last_undegraded":"2024-09-13T00:52:40.261060+0000","last_fullsized":"2024-09-13T00:52:40.261060+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:35:27.422911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"63'3","reported_seq":943,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.321992+0000","last_change":"2024-09-13T00:46:45.516465+0000","last_active":"2024-09-13T00:52:40.321992+0000","last_peered":"2024-09-13T00:52:40.321992+0000","last_clean":"2024-09-13T00:52:40.321992+0000","last_became_active":"2024-09-13T00:46:45.515275+0000","last_became_peered":"2024-09-13T00:46:45.515275+0000","last_unstale":"2024-09-13T00:52:40.321992+0000","last_undegraded":"2024-09-13T00:52:40.321992+0000","last_fullsized":"2024-09-13T00:52:40.321992+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:44:06.608978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":929,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.276547+0000","last_change":"2024-09-13T00:46:45.519171+0000","last_active":"2024-09-13T00:52:39.276547+0000","last_peered":"2024-09-13T00:52:39.276547+0000","last_clean":"2024-09-13T00:52:39.276547+0000","last_became_active":"2024-09-13T00:46:45.518674+0000","last_became_peered":"2024-09-13T00:46:45.518674+0000","last_unstale":"2024-09-13T00:52:39.276547+0000","last_undegraded":"2024-09-13T00:52:39.276547+0000","last_fullsized":"2024-09-13T00:52:39.276547+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:43:10.733090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":910,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.258570+0000","last_change":"2024-09-13T00:46:15.327439+0000","last_active":"2024-09-13T00:52:39.258570+0000","last_peered":"2024-09-13T00:52:39.258570+0000","last_clean":"2024-09-13T00:52:39.258570+0000","last_became_active":"2024-09-13T00:46:15.326691+0000","last_became_peered":"2024-09-13T00:46:15.326691+0000","last_unstale":"2024-09-13T00:52:39.258570+0000","last_undegraded":"2024-09-13T00:52:39.258570+0000","last_fullsized":"2024-09-13T00:52:39.258570+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:52:52.553904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":915,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.431162+0000","last_change":"2024-09-13T00:43:56.097526+0000","last_active":"2024-09-13T00:52:39.431162+0000","last_peered":"2024-09-13T00:52:39.431162+0000","last_clean":"2024-09-13T00:52:39.431162+0000","last_became_active":"2024-09-13T00:43:56.096340+0000","last_became_peered":"2024-09-13T00:43:56.096340+0000","last_unstale":"2024-09-13T00:52:39.431162+0000","last_undegraded":"2024-09-13T00:52:39.431162+0000","last_fullsized":"2024-09-13T00:52:39.431162+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:41:33.452908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"62'1","reported_seq":898,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.280351+0000","last_change":"2024-09-13T00:32:23.122340+0000","last_active":"2024-09-13T00:52:39.280351+0000","last_peered":"2024-09-13T00:52:39.280351+0000","last_clean":"2024-09-13T00:52:39.280351+0000","last_became_active":"2024-09-13T00:32:23.121795+0000","last_became_peered":"2024-09-13T00:32:23.121795+0000","last_unstale":"2024-09-13T00:52:39.280351+0000","last_undegraded":"2024-09-13T00:52:39.280351+0000","last_fullsized":"2024-09-13T00:52:39.280351+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:18:28.661306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":937,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.260382+0000","last_change":"2024-09-13T00:46:45.515627+0000","last_active":"2024-09-13T00:52:40.260382+0000","last_peered":"2024-09-13T00:52:40.260382+0000","last_clean":"2024-09-13T00:52:40.260382+0000","last_became_active":"2024-09-13T00:46:45.509541+0000","last_became_peered":"2024-09-13T00:46:45.509541+0000","last_unstale":"2024-09-13T00:52:40.260382+0000","last_undegraded":"2024-09-13T00:52:40.260382+0000","last_fullsized":"2024-09-13T00:52:40.260382+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:46:15.698098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"62'2","reported_seq":933,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.280246+0000","last_change":"2024-09-13T00:46:45.523694+0000","last_active":"2024-09-13T00:52:39.280246+0000","last_peered":"2024-09-13T00:52:39.280246+0000","last_clean":"2024-09-13T00:52:39.280246+0000","last_became_active":"2024-09-13T00:46:45.523318+0000","last_became_peered":"2024-09-13T00:46:45.523318+0000","last_unstale":"2024-09-13T00:52:39.280246+0000","last_undegraded":"2024-09-13T00:52:39.280246+0000","last_fullsized":"2024-09-13T00:52:39.280246+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:39:17.203202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":892,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.258169+0000","last_change":"2024-09-13T00:32:23.861007+0000","last_active":"2024-09-13T00:52:39.258169+0000","last_peered":"2024-09-13T00:52:39.258169+0000","last_clean":"2024-09-13T00:52:39.258169+0000","last_became_active":"2024-09-13T00:32:23.860213+0000","last_became_peered":"2024-09-13T00:32:23.860213+0000","last_unstale":"2024-09-13T00:52:39.258169+0000","last_undegraded":"2024-09-13T00:52:39.258169+0000","last_fullsized":"2024-09-13T00:52:39.258169+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:58:20.866610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"62'1","reported_seq":932,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.280199+0000","last_change":"2024-09-13T00:46:45.502612+0000","last_active":"2024-09-13T00:52:39.280199+0000","last_peered":"2024-09-13T00:52:39.280199+0000","last_clean":"2024-09-13T00:52:39.280199+0000","last_became_active":"2024-09-13T00:46:45.502062+0000","last_became_peered":"2024-09-13T00:46:45.502062+0000","last_unstale":"2024-09-13T00:52:39.280199+0000","last_undegraded":"2024-09-13T00:52:39.280199+0000","last_fullsized":"2024-09-13T00:52:39.280199+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:25:34.771467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"62'8","reported_seq":928,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.430964+0000","last_change":"2024-09-13T00:43:18.190299+0000","last_active":"2024-09-13T00:52:39.430964+0000","last_peered":"2024-09-13T00:52:39.430964+0000","last_clean":"2024-09-13T00:52:39.430964+0000","last_became_active":"2024-09-13T00:43:18.189192+0000","last_became_peered":"2024-09-13T00:43:18.189192+0000","last_unstale":"2024-09-13T00:52:39.430964+0000","last_undegraded":"2024-09-13T00:52:39.430964+0000","last_fullsized":"2024-09-13T00:52:39.430964+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:02:26.977888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":923,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.320939+0000","last_change":"2024-09-13T00:46:15.349052+0000","last_active":"2024-09-13T00:52:40.320939+0000","last_peered":"2024-09-13T00:52:40.320939+0000","last_clean":"2024-09-13T00:52:40.320939+0000","last_became_active":"2024-09-13T00:46:15.346970+0000","last_became_peered":"2024-09-13T00:46:15.346970+0000","last_unstale":"2024-09-13T00:52:40.320939+0000","last_undegraded":"2024-09-13T00:52:40.320939+0000","last_fullsized":"2024-09-13T00:52:40.320939+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:34:13.110080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":913,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.279467+0000","last_change":"2024-09-13T00:46:45.508486+0000","last_active":"2024-09-13T00:52:39.279467+0000","last_peered":"2024-09-13T00:52:39.279467+0000","last_clean":"2024-09-13T00:52:39.279467+0000","last_became_active":"2024-09-13T00:46:45.508135+0000","last_became_peered":"2024-09-13T00:46:45.508135+0000","last_unstale":"2024-09-13T00:52:39.279467+0000","last_undegraded":"2024-09-13T00:52:39.279467+0000","last_fullsized":"2024-09-13T00:52:39.279467+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:05:04.774232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":900,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.319010+0000","last_change":"2024-09-13T00:46:15.332428+0000","last_active":"2024-09-13T00:52:40.319010+0000","last_peered":"2024-09-13T00:52:40.319010+0000","last_clean":"2024-09-13T00:52:40.319010+0000","last_became_active":"2024-09-13T00:46:15.331967+0000","last_became_peered":"2024-09-13T00:46:15.331967+0000","last_unstale":"2024-09-13T00:52:40.319010+0000","last_undegraded":"2024-09-13T00:52:40.319010+0000","last_fullsized":"2024-09-13T00:52:40.319010+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:02:07.306591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":900,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.276229+0000","last_change":"2024-09-13T00:46:15.356215+0000","last_active":"2024-09-13T00:52:39.276229+0000","last_peered":"2024-09-13T00:52:39.276229+0000","last_clean":"2024-09-13T00:52:39.276229+0000","last_became_active":"2024-09-13T00:46:15.355427+0000","last_became_peered":"2024-09-13T00:46:15.355427+0000","last_unstale":"2024-09-13T00:52:39.276229+0000","last_undegraded":"2024-09-13T00:52:39.276229+0000","last_fullsized":"2024-09-13T00:52:39.276229+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:39:06.852517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":909,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.259364+0000","last_change":"2024-09-13T00:46:15.329535+0000","last_active":"2024-09-13T00:52:39.259364+0000","last_peered":"2024-09-13T00:52:39.259364+0000","last_clean":"2024-09-13T00:52:39.259364+0000","last_became_active":"2024-09-13T00:46:15.329152+0000","last_became_peered":"2024-09-13T00:46:15.329152+0000","last_unstale":"2024-09-13T00:52:39.259364+0000","last_undegraded":"2024-09-13T00:52:39.259364+0000","last_fullsized":"2024-09-13T00:52:39.259364+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:40:36.773015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"62'2","reported_seq":934,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.279892+0000","last_change":"2024-09-13T00:46:45.504711+0000","last_active":"2024-09-13T00:52:39.279892+0000","last_peered":"2024-09-13T00:52:39.279892+0000","last_clean":"2024-09-13T00:52:39.279892+0000","last_became_active":"2024-09-13T00:46:45.504345+0000","last_became_peered":"2024-09-13T00:46:45.504345+0000","last_unstale":"2024-09-13T00:52:39.279892+0000","last_undegraded":"2024-09-13T00:52:39.279892+0000","last_fullsized":"2024-09-13T00:52:39.279892+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:34:15.943161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":909,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.320699+0000","last_change":"2024-09-13T00:46:15.350681+0000","last_active":"2024-09-13T00:52:40.320699+0000","last_peered":"2024-09-13T00:52:40.320699+0000","last_clean":"2024-09-13T00:52:40.320699+0000","last_became_active":"2024-09-13T00:46:15.350293+0000","last_became_peered":"2024-09-13T00:46:15.350293+0000","last_unstale":"2024-09-13T00:52:40.320699+0000","last_undegraded":"2024-09-13T00:52:40.320699+0000","last_fullsized":"2024-09-13T00:52:40.320699+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T12:31:39.094477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":414,"num_read_kb":849,"num_write":173,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112,"ondisk_log_size":112,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5536,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":0},"log_size":62,"ondisk_log_size":62,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430520,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":321744,"kb_used_data":16068,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93423408,"statfs":{"total":95995035648,"available":95665569792,"internally_reserved":0,"allocated":16453632,"data_stored":10508274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:53:47 2024","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.83999999999999997,"15min":0.84499999999999997},"min":{"1min":0.52200000000000002,"5min":0.44700000000000001,"15min":0.38},"max":{"1min":1.167,"5min":1.2529999999999999,"15min":2.8250000000000002},"last":0.97699999999999998},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.1379999999999999,"15min":1.0229999999999999},"min":{"1min":0.53300000000000003,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.21,"5min":16.599,"15min":16.599},"last":0.94599999999999995}]},{"osd":1,"last update":"Fri Sep 13 00:54:09 2024","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.0669999999999999,"15min":1.0840000000000001},"min":{"1min":0.64100000000000001,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.286,"5min":1.53,"15min":3.9239999999999999},"last":1.181},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.0760000000000001,"15min":1.0369999999999999},"min":{"1min":0.64200000000000002,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.3220000000000001,"5min":1.5880000000000001,"15min":1.5880000000000001},"last":1.1619999999999999}]},{"osd":2,"last update":"Fri Sep 13 00:53:53 2024","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.99099999999999999,"15min":0.91300000000000003},"min":{"1min":0.71499999999999997,"5min":0.54500000000000004,"15min":0.53900000000000003},"max":{"1min":1.2330000000000001,"5min":2.0760000000000001,"15min":2.0760000000000001},"last":1.206},{"interface":"front","average":{"1min":0.98799999999999999,"5min":1.032,"15min":0.98599999999999999},"min":{"1min":0.72999999999999998,"5min":0.377,"15min":0.377},"max":{"1min":1.323,"5min":1.458,"15min":3.8479999999999999},"last":0.84599999999999997}]},{"osd":3,"last update":"Fri Sep 13 00:53:25 2024","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.94399999999999995,"15min":0.94599999999999995},"min":{"1min":0.69799999999999995,"5min":0.56399999999999995,"15min":0.48499999999999999},"max":{"1min":1.1000000000000001,"5min":1.97,"15min":2.96},"last":0.91400000000000003},{"interface":"front","average":{"1min":1.016,"5min":1.089,"15min":1.0269999999999999},"min":{"1min":0.72399999999999998,"5min":0.623,"15min":0.58599999999999997},"max":{"1min":1.282,"5min":2.3359999999999999,"15min":2.3359999999999999},"last":1.0940000000000001}]},{"osd":4,"last update":"Fri Sep 13 00:53:53 2024","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.0289999999999999,"15min":0.99299999999999999},"min":{"1min":0.83299999999999996,"5min":0.60099999999999998,"15min":0.54500000000000004},"max":{"1min":1.3100000000000001,"5min":2.4319999999999999,"15min":2.4319999999999999},"last":0.88300000000000001},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.077,"15min":1.1699999999999999},"min":{"1min":0.68600000000000005,"5min":0.58599999999999997,"15min":0.56399999999999995},"max":{"1min":1.365,"5min":3.2530000000000001,"15min":3.6579999999999999},"last":1.1339999999999999}]},{"osd":5,"last update":"Fri Sep 13 00:54:09 2024","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.075,"15min":1.0209999999999999},"min":{"1min":0.78100000000000003,"5min":0.621,"15min":0.38800000000000001},"max":{"1min":1.3149999999999999,"5min":2.863,"15min":2.863},"last":1.024},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.1879999999999999,"15min":1.103},"min":{"1min":0.78900000000000003,"5min":0.59799999999999998,"15min":0.46000000000000002},"max":{"1min":1.3049999999999999,"5min":5.4960000000000004,"15min":5.4960000000000004},"last":1.004}]},{"osd":6,"last update":"Fri Sep 13 00:54:09 2024","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.2030000000000001,"15min":1.173},"min":{"1min":0.96799999999999997,"5min":0.67100000000000004,"15min":0.36899999999999999},"max":{"1min":1.3520000000000001,"5min":2.2970000000000002,"15min":5.8849999999999998},"last":1.0649999999999999},{"interface":"front","average":{"1min":1.032,"5min":1.0169999999999999,"15min":1.0760000000000001},"min":{"1min":0.72899999999999998,"5min":0.51100000000000001,"15min":0.40699999999999997},"max":{"1min":1.339,"5min":1.413,"15min":5.6349999999999998},"last":0.73199999999999998}]}]},{"osd":6,"up_from":42,"seq":180388626754,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":336892,"kb_used_data":16052,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93408260,"statfs":{"total":95995035648,"available":95650058240,"internally_reserved":0,"allocated":16437248,"data_stored":10507588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:53:35 2024","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.026,"15min":0.97899999999999998},"min":{"1min":0.54400000000000004,"5min":0.443,"15min":0.33700000000000002},"max":{"1min":1.4450000000000001,"5min":4.3140000000000001,"15min":8.3529999999999998},"last":0.68600000000000005},{"interface":"front","average":{"1min":1.0329999999999999,"5min":0.97699999999999998,"15min":0.97199999999999998},"min":{"1min":0.45800000000000002,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.55,"5min":4.4160000000000004,"15min":8.423},"last":1.2050000000000001}]},{"osd":1,"last update":"Fri Sep 13 00:53:16 2024","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.98999999999999999,"15min":0.96099999999999997},"min":{"1min":0.51800000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.593,"5min":5.0010000000000003,"15min":5.0010000000000003},"last":0.73499999999999999},{"interface":"front","average":{"1min":1.0900000000000001,"5min":1.1200000000000001,"15min":1.0349999999999999},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":3.2639999999999998,"5min":8.1210000000000004,"15min":8.1210000000000004},"last":0.84199999999999997}]},{"osd":2,"last update":"Fri Sep 13 00:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":1.1000000000000001,"15min":1.304},"min":{"1min":0.443,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.415,"5min":7.9489999999999998,"15min":15.148999999999999},"last":1.048},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.175,"15min":1.006},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.333,"5min":8.2040000000000006,"15min":8.2040000000000006},"last":0.93600000000000005}]},{"osd":3,"last update":"Fri Sep 13 00:53:29 2024","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":1.022,"15min":1.266},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.48199999999999998},"max":{"1min":1.1870000000000001,"5min":4.5060000000000002,"15min":8.8490000000000002},"last":0.78800000000000003},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.341,"15min":1.274},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.44},"max":{"1min":1.397,"5min":11.970000000000001,"15min":11.970000000000001},"last":1.0049999999999999}]},{"osd":4,"last update":"Fri Sep 13 00:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.93700000000000006,"15min":0.97299999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.0840000000000001,"5min":5.4219999999999997,"15min":5.4219999999999997},"last":1.1259999999999999},{"interface":"front","average":{"1min":1.131,"5min":1.2030000000000001,"15min":1.145},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":1.3440000000000001,"5min":8.4160000000000004,"15min":8.4160000000000004},"last":1.2410000000000001}]},{"osd":5,"last update":"Fri Sep 13 00:53:35 2024","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":1.2010000000000001,"15min":1.115},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.2190000000000001,"5min":18.739999999999998,"15min":18.739999999999998},"last":0.96899999999999997},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.236,"15min":1.1499999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.32100000000000001},"max":{"1min":1.5189999999999999,"5min":8.3439999999999994,"15min":12.009},"last":1.1679999999999999}]},{"osd":7,"last update":"Fri Sep 13 00:53:21 2024","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.1259999999999999,"15min":1.171},"min":{"1min":0.48599999999999999,"5min":0.40999999999999998,"15min":0.379},"max":{"1min":1.7070000000000001,"5min":4.9130000000000003,"15min":20.638000000000002},"last":1.0860000000000001},{"interface":"front","average":{"1min":1.155,"5min":1.242,"15min":1.218},"min":{"1min":0.433,"5min":0.433,"15min":0.42799999999999999},"max":{"1min":1.9399999999999999,"5min":4.766,"15min":9.7750000000000004},"last":1.2709999999999999}]}]},{"osd":5,"up_from":36,"seq":154618822988,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":366880,"kb_used_data":15464,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93378272,"statfs":{"total":95995035648,"available":95619350528,"internally_reserved":0,"allocated":15835136,"data_stored":9913040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:53:41 2024","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":1.038,"15min":0.93200000000000005},"min":{"1min":0.66000000000000003,"5min":0.5,"15min":0.46700000000000003},"max":{"1min":1.25,"5min":5.9050000000000002,"15min":5.9050000000000002},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.88800000000000001,"5min":1.0129999999999999,"15min":1},"min":{"1min":0.629,"5min":0.48999999999999999,"15min":0.48399999999999999},"max":{"1min":1.0880000000000001,"5min":5.79,"15min":8.5899999999999999},"last":1.004}]},{"osd":1,"last update":"Fri Sep 13 00:53:12 2024","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":1.008,"15min":1.155},"min":{"1min":0.68600000000000005,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.21,"5min":6.0300000000000002,"15min":7.835},"last":0.97299999999999998},{"interface":"front","average":{"1min":0.97699999999999998,"5min":1.077,"15min":1.2450000000000001},"min":{"1min":0.73899999999999999,"5min":0.443,"15min":0.443},"max":{"1min":1.411,"5min":5.5270000000000001,"15min":8.8599999999999994},"last":0.83799999999999997}]},{"osd":2,"last update":"Fri Sep 13 00:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.056,"15min":0.98599999999999999},"min":{"1min":0.66800000000000004,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.2330000000000001,"5min":5.8380000000000001,"15min":16.137},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.92400000000000004,"5min":1.131,"15min":1.008},"min":{"1min":0.71499999999999997,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.129,"5min":11.856,"15min":11.856},"last":0.94999999999999996}]},{"osd":3,"last update":"Fri Sep 13 00:53:23 2024","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.0680000000000001,"15min":1.02},"min":{"1min":0.60199999999999998,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.2629999999999999,"5min":4.0439999999999996,"15min":5.9509999999999996},"last":0.71399999999999997},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.2,"15min":1.1579999999999999},"min":{"1min":0.752,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.2509999999999999,"5min":5.5229999999999997,"15min":10.887},"last":0.80100000000000005}]},{"osd":4,"last update":"Fri Sep 13 00:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":1.048,"15min":1.0960000000000001},"min":{"1min":0.64000000000000001,"5min":0.54900000000000004,"15min":0.54900000000000004},"max":{"1min":1.222,"5min":13.478999999999999,"15min":13.478999999999999},"last":0.871},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.139,"15min":1.179},"min":{"1min":0.66500000000000004,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.286,"5min":5.7469999999999999,"15min":5.7469999999999999},"last":0.90200000000000002}]},{"osd":6,"last update":"Fri Sep 13 00:53:23 2024","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.2929999999999999,"15min":1.125},"min":{"1min":0.66500000000000004,"5min":0.629,"15min":0.34100000000000003},"max":{"1min":1.4370000000000001,"5min":12.81,"15min":12.81},"last":1.02},{"interface":"front","average":{"1min":1.139,"5min":1.3799999999999999,"15min":1.218},"min":{"1min":0.873,"5min":0.65500000000000003,"15min":0.36199999999999999},"max":{"1min":1.4390000000000001,"5min":12.98,"15min":12.98},"last":0.93200000000000005}]},{"osd":7,"last update":"Fri Sep 13 00:53:12 2024","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.4039999999999999,"15min":1.306},"min":{"1min":0.75800000000000001,"5min":0.68400000000000005,"15min":0.39700000000000002},"max":{"1min":1.3939999999999999,"5min":13.388999999999999,"15min":13.388999999999999},"last":1.0620000000000001},{"interface":"front","average":{"1min":1.2629999999999999,"5min":1.3480000000000001,"15min":1.196},"min":{"1min":0.67300000000000004,"5min":0.67100000000000004,"15min":0.42599999999999999},"max":{"1min":2.46,"5min":12.470000000000001,"15min":12.470000000000001},"last":1.042}]}]},{"osd":4,"up_from":649,"seq":2787433775195,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":69444,"kb_used_data":15484,"kb_used_omap":67,"kb_used_meta":53884,"kb_avail":93675708,"statfs":{"total":95995035648,"available":95923924992,"internally_reserved":0,"allocated":15855616,"data_stored":9917836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68780,"internal_metadata":55178068},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.97799999999999998,"15min":1.0489999999999999},"min":{"1min":0.55600000000000005,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.1859999999999999,"5min":3.8359999999999999,"15min":4.7279999999999998},"last":1.1120000000000001},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.0049999999999999,"15min":1.0600000000000001},"min":{"1min":0.49199999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.27,"5min":3.6779999999999999,"15min":3.6779999999999999},"last":1.175}]},{"osd":1,"last update":"Fri Sep 13 00:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.0089999999999999,"15min":0.996},"min":{"1min":0.73899999999999999,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.3300000000000001,"5min":4.8390000000000004,"15min":6.9649999999999999},"last":1.2769999999999999},{"interface":"front","average":{"1min":0.89600000000000002,"5min":1.0289999999999999,"15min":1.0860000000000001},"min":{"1min":0.63500000000000001,"5min":0.58499999999999996,"15min":0.56799999999999995},"max":{"1min":1.2170000000000001,"5min":3.4159999999999999,"15min":5.7240000000000002},"last":0.79200000000000004}]},{"osd":2,"last update":"Fri Sep 13 00:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.98499999999999999,"15min":1.04},"min":{"1min":0.61099999999999999,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.2589999999999999,"5min":2.8650000000000002,"15min":6.9349999999999996},"last":0.99299999999999999},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.121,"15min":1.1160000000000001},"min":{"1min":0.55500000000000005,"5min":0.45700000000000002,"15min":0.41599999999999998},"max":{"1min":1.2270000000000001,"5min":6.5810000000000004,"15min":6.5810000000000004},"last":1.0740000000000001}]},{"osd":3,"last update":"Fri Sep 13 00:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.0529999999999999,"15min":1.127},"min":{"1min":0.58499999999999996,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.2130000000000001,"5min":3.7639999999999998,"15min":3.7639999999999998},"last":0.95899999999999996},{"interface":"front","average":{"1min":0.92300000000000004,"5min":1.042,"15min":1.0640000000000001},"min":{"1min":0.64300000000000002,"5min":0.379,"15min":0.379},"max":{"1min":1.216,"5min":7.7889999999999997,"15min":7.7889999999999997},"last":1.018}]},{"osd":5,"last update":"Fri Sep 13 00:53:58 2024","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.0620000000000001,"15min":1.075},"min":{"1min":0.72099999999999997,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.3440000000000001,"5min":10.381,"15min":10.381},"last":0.92700000000000005},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.034,"15min":1.046},"min":{"1min":0.80900000000000005,"5min":0.374,"15min":0.374},"max":{"1min":1.3799999999999999,"5min":5.0259999999999998,"15min":5.0259999999999998},"last":1.2090000000000001}]},{"osd":6,"last update":"Fri Sep 13 00:53:58 2024","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.147,"15min":1.0980000000000001},"min":{"1min":0.77000000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.46,"5min":10.474,"15min":10.474},"last":1.1359999999999999},{"interface":"front","average":{"1min":1.157,"5min":1.157,"15min":1.1679999999999999},"min":{"1min":0.85599999999999998,"5min":0.433,"15min":0.433},"max":{"1min":1.4410000000000001,"5min":4.9459999999999997,"15min":6.6159999999999997},"last":0.877}]},{"osd":7,"last update":"Fri Sep 13 00:53:58 2024","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.125,"15min":1.21},"min":{"1min":0.68799999999999994,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.502,"5min":3.2749999999999999,"15min":3.3940000000000001},"last":0.69499999999999995},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.2649999999999999,"15min":1.2609999999999999},"min":{"1min":0.878,"5min":0.505,"15min":0.505},"max":{"1min":1.407,"5min":10.548,"15min":12.227},"last":1.2529999999999999}]}]},{"osd":3,"up_from":643,"seq":2761663971425,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":72244,"kb_used_data":15464,"kb_used_omap":74,"kb_used_meta":45685,"kb_avail":93672908,"statfs":{"total":95995035648,"available":95921057792,"internally_reserved":0,"allocated":15835136,"data_stored":9913030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76118,"internal_metadata":46782122},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:53:35 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.82899999999999996,"15min":3.4780000000000002},"min":{"1min":0.33300000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.93400000000000005,"5min":14.706,"15min":92.156000000000006},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.75900000000000001,"5min":0.81200000000000006,"15min":2.9129999999999998},"min":{"1min":0.48199999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.1899999999999999,"5min":7.4950000000000001,"15min":72.637},"last":1.3580000000000001}]},{"osd":1,"last update":"Fri Sep 13 00:53:35 2024","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.78000000000000003,"15min":3.1080000000000001},"min":{"1min":0.36899999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.0069999999999999,"5min":1.5169999999999999,"15min":75.415999999999997},"last":1.5860000000000001},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.73399999999999999,"15min":2.4079999999999999},"min":{"1min":0.441,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.3109999999999999,"5min":2.0139999999999998,"15min":59.807000000000002},"last":1.4039999999999999}]},{"osd":2,"last update":"Fri Sep 13 00:53:35 2024","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.83499999999999996,"15min":2.0640000000000001},"min":{"1min":0.45300000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.044,"5min":3.1930000000000001,"15min":26.347999999999999},"last":0.83599999999999997},{"interface":"front","average":{"1min":0.878,"5min":0.96499999999999997,"15min":3.2010000000000001},"min":{"1min":0.51500000000000001,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.153,"5min":3.3860000000000001,"15min":75.534000000000006},"last":1.5069999999999999}]},{"osd":4,"last update":"Fri Sep 13 00:54:01 2024","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.0429999999999999,"15min":1.466},"min":{"1min":0.65400000000000003,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.2210000000000001,"5min":3.391,"15min":9.6010000000000009},"last":1.1539999999999999},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.028,"15min":1.3009999999999999},"min":{"1min":0.60399999999999998,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.2809999999999999,"5min":2.2210000000000001,"15min":9.3070000000000004},"last":1.3220000000000001}]},{"osd":5,"last update":"Fri Sep 13 00:53:35 2024","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.0089999999999999,"15min":2.7429999999999999},"min":{"1min":0.60399999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.28,"5min":2.9100000000000001,"15min":59.741},"last":1.464},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.1319999999999999,"15min":2.9119999999999999},"min":{"1min":0.66200000000000003,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.5109999999999999,"5min":9.8919999999999995,"15min":60.402000000000001},"last":1.083}]},{"osd":6,"last update":"Fri Sep 13 00:53:35 2024","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.075,"15min":3.2839999999999998},"min":{"1min":0.55300000000000005,"5min":0.44,"15min":0.44},"max":{"1min":1.4399999999999999,"5min":1.8340000000000001,"15min":75.468999999999994},"last":0.95499999999999996},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.1200000000000001,"15min":3.3980000000000001},"min":{"1min":0.746,"5min":0.499,"15min":0.499},"max":{"1min":1.365,"5min":3.46,"15min":75.626999999999995},"last":1.663}]},{"osd":7,"last update":"Fri Sep 13 00:53:35 2024","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.0920000000000001,"15min":1.702},"min":{"1min":0.63700000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.333,"5min":2.266,"15min":16.564},"last":1.4370000000000001},{"interface":"front","average":{"1min":1.091,"5min":1.1799999999999999,"15min":4.0720000000000001},"min":{"1min":0.76100000000000001,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.4810000000000001,"5min":5.1289999999999996,"15min":92.200999999999993},"last":1.02}]}]},{"osd":2,"up_from":629,"seq":2701534429288,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":105976,"kb_used_data":15464,"kb_used_omap":72,"kb_used_meta":58167,"kb_avail":93639176,"statfs":{"total":95995035648,"available":95886516224,"internally_reserved":0,"allocated":15835136,"data_stored":9909549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74464,"internal_metadata":59563296},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:53:52 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.83299999999999996,"15min":0.66600000000000004},"min":{"1min":0.40100000000000002,"5min":0.224,"15min":0.20000000000000001},"max":{"1min":0.89400000000000002,"5min":13.68,"15min":13.68},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.97199999999999998,"15min":0.79800000000000004},"min":{"1min":0.36399999999999999,"5min":0.27500000000000002,"15min":0.22800000000000001},"max":{"1min":0.88800000000000001,"5min":15.130000000000001,"15min":15.130000000000001},"last":0.80600000000000005}]},{"osd":1,"last update":"Fri Sep 13 00:53:52 2024","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.79400000000000004,"15min":0.69699999999999995},"min":{"1min":0.42199999999999999,"5min":0.314,"15min":0.249},"max":{"1min":1.032,"5min":4.0940000000000003,"15min":4.0940000000000003},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.877,"15min":0.76600000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.26600000000000001},"max":{"1min":1.095,"5min":6.3029999999999999,"15min":6.3029999999999999},"last":0.68000000000000005}]},{"osd":3,"last update":"Fri Sep 13 00:53:28 2024","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.86099999999999999,"15min":0.84699999999999998},"min":{"1min":0.40999999999999998,"5min":0.36599999999999999,"15min":0.29099999999999998},"max":{"1min":1.1950000000000001,"5min":2.573,"15min":11.715999999999999},"last":1.2430000000000001},{"interface":"front","average":{"1min":0.91700000000000004,"5min":0.95599999999999996,"15min":0.85699999999999998},"min":{"1min":0.46500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.0379999999999998,"5min":3.0699999999999998,"15min":3.504},"last":0.71999999999999997}]},{"osd":4,"last update":"Fri Sep 13 00:53:52 2024","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.0940000000000001,"15min":1.52},"min":{"1min":0.68000000000000005,"5min":0.55800000000000005,"15min":0.47199999999999998},"max":{"1min":1.244,"5min":4.3949999999999996,"15min":12.356999999999999},"last":1.077},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.226,"15min":1.234},"min":{"1min":0.64900000000000002,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.2889999999999999,"5min":10.207000000000001,"15min":10.207000000000001},"last":0.94299999999999995}]},{"osd":5,"last update":"Fri Sep 13 00:53:52 2024","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.1479999999999999,"15min":1.022},"min":{"1min":0.63700000000000001,"5min":0.58799999999999997,"15min":0.47899999999999998},"max":{"1min":1.2070000000000001,"5min":6.4630000000000001,"15min":7.625},"last":1.026},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.234,"15min":1.091},"min":{"1min":0.65700000000000003,"5min":0.44600000000000001,"15min":0.40500000000000003},"max":{"1min":1.2849999999999999,"5min":15.185,"15min":15.185},"last":1.1699999999999999}]},{"osd":6,"last update":"Fri Sep 13 00:53:52 2024","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.238,"15min":1.101},"min":{"1min":0.66300000000000003,"5min":0.57799999999999996,"15min":0.44800000000000001},"max":{"1min":1.2430000000000001,"5min":6.1529999999999996,"15min":12.483000000000001},"last":1.3169999999999999},{"interface":"front","average":{"1min":1.036,"5min":1.27,"15min":1.0880000000000001},"min":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.53400000000000003},"max":{"1min":1.3320000000000001,"5min":7.9480000000000004,"15min":7.9480000000000004},"last":1.0089999999999999}]},{"osd":7,"last update":"Fri Sep 13 00:53:52 2024","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.3129999999999999,"15min":1.103},"min":{"1min":0.64500000000000002,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.3160000000000001,"5min":11.964,"15min":11.964},"last":1.048},{"interface":"front","average":{"1min":1.036,"5min":1.2110000000000001,"15min":1.119},"min":{"1min":0.67800000000000005,"5min":0.56699999999999995,"15min":0.53500000000000003},"max":{"1min":1.2430000000000001,"5min":4.7149999999999999,"15min":7.5350000000000001},"last":0.97099999999999997}]}]},{"osd":1,"up_from":539,"seq":2314987372669,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":69292,"kb_used_data":15456,"kb_used_omap":53,"kb_used_meta":53770,"kb_avail":93675860,"statfs":{"total":95995035648,"available":95924080640,"internally_reserved":0,"allocated":15826944,"data_stored":9906981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":54760,"internal_metadata":55061016},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:53:16 2024","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":0.72499999999999998,"15min":0.73299999999999998},"min":{"1min":0.29699999999999999,"5min":0.25600000000000001,"15min":0.252},"max":{"1min":1.3009999999999999,"5min":2.056,"15min":2.778},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.79600000000000004,"15min":0.80200000000000005},"min":{"1min":0.439,"5min":0.29799999999999999,"15min":0.29399999999999998},"max":{"1min":1.629,"5min":2.4399999999999999,"15min":9.625},"last":0.95199999999999996}]},{"osd":2,"last update":"Fri Sep 13 00:53:51 2024","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.71599999999999997,"15min":0.67900000000000005},"min":{"1min":0.39300000000000002,"5min":0.27400000000000002,"15min":0.20399999999999999},"max":{"1min":1.395,"5min":1.4490000000000001,"15min":2.7330000000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.80200000000000005,"15min":0.73799999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":1.0169999999999999,"5min":1.399,"15min":2.2629999999999999},"last":0.878}]},{"osd":3,"last update":"Fri Sep 13 00:53:29 2024","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.85999999999999999,"15min":0.92300000000000004},"min":{"1min":0.441,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.345,"5min":1.345,"15min":5.2300000000000004},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.90500000000000003,"15min":0.89200000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.4319999999999999,"5min":1.4870000000000001,"15min":1.889},"last":1.1659999999999999}]},{"osd":4,"last update":"Fri Sep 13 00:54:05 2024","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":1.018,"15min":1.6910000000000001},"min":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.54400000000000004},"max":{"1min":1.2310000000000001,"5min":1.641,"15min":16.684999999999999},"last":1.1990000000000001},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.1100000000000001,"15min":1.8280000000000001},"min":{"1min":0.58499999999999996,"5min":0.48699999999999999,"15min":0.48099999999999998},"max":{"1min":4.1820000000000004,"5min":5.9100000000000001,"15min":24.937999999999999},"last":1.0669999999999999}]},{"osd":5,"last update":"Fri Sep 13 00:53:16 2024","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.353,"15min":1.163},"min":{"1min":0.56399999999999995,"5min":0.53700000000000003,"15min":0.42299999999999999},"max":{"1min":3.673,"5min":29.626000000000001,"15min":29.626000000000001},"last":1.1180000000000001},{"interface":"front","average":{"1min":1.107,"5min":1.1559999999999999,"15min":1.123},"min":{"1min":0.92800000000000005,"5min":0.64100000000000001,"15min":0.498},"max":{"1min":2.3330000000000002,"5min":3.9390000000000001,"15min":5.7930000000000001},"last":1.101}]},{"osd":6,"last update":"Fri Sep 13 00:53:16 2024","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.0800000000000001,"15min":1.1160000000000001},"min":{"1min":0.63200000000000001,"5min":0.501,"15min":0.501},"max":{"1min":1.728,"5min":2.7290000000000001,"15min":9.2550000000000008},"last":1.226},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.2609999999999999,"15min":1.202},"min":{"1min":0.81699999999999995,"5min":0.70599999999999996,"15min":0.52100000000000002},"max":{"1min":1.4339999999999999,"5min":12.065,"15min":12.065},"last":1.258}]},{"osd":7,"last update":"Fri Sep 13 00:53:16 2024","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":1.3759999999999999,"15min":1.2769999999999999},"min":{"1min":0.95999999999999996,"5min":0.85599999999999998,"15min":0.497},"max":{"1min":12.346,"5min":12.346,"15min":12.346},"last":1.01},{"interface":"front","average":{"1min":1.262,"5min":1.286,"15min":1.266},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.53200000000000003},"max":{"1min":4.4189999999999996,"5min":7.1050000000000004,"15min":7.1050000000000004},"last":1.28}]}]},{"osd":0,"up_from":502,"seq":2156073582725,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":111724,"kb_used_data":16048,"kb_used_omap":55,"kb_used_meta":53832,"kb_avail":93633428,"statfs":{"total":95995035648,"available":95880630272,"internally_reserved":0,"allocated":16433152,"data_stored":10503900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56531,"internal_metadata":55124781},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 00:53:15 2024","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":1.0069999999999999,"15min":0.875},"min":{"1min":0.443,"5min":0.41799999999999998,"15min":0.30199999999999999},"max":{"1min":1.1279999999999999,"5min":24.273,"15min":24.273},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.85999999999999999,"5min":0.85899999999999999,"15min":0.872},"min":{"1min":0.63300000000000001,"5min":0.503,"15min":0.29199999999999998},"max":{"1min":1.181,"5min":1.7,"15min":11.792999999999999},"last":0.76400000000000001}]},{"osd":2,"last update":"Fri Sep 13 00:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.78100000000000003,"15min":0.92800000000000005},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28299999999999997},"max":{"1min":1.075,"5min":1.7230000000000001,"15min":8.4969999999999999},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.77200000000000002,"15min":1.5880000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":1.4399999999999999,"5min":2.2040000000000002,"15min":32.914000000000001},"last":0.998}]},{"osd":3,"last update":"Fri Sep 13 00:53:27 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.89300000000000002,"15min":1.0489999999999999},"min":{"1min":0.65800000000000003,"5min":0.48199999999999998,"15min":0.33900000000000002},"max":{"1min":1.647,"5min":1.7689999999999999,"15min":8.7520000000000007},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.81200000000000006,"5min":1.1559999999999999,"15min":1.1850000000000001},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.309},"max":{"1min":1.226,"5min":24.571999999999999,"15min":24.571999999999999},"last":0.81100000000000005}]},{"osd":4,"last update":"Fri Sep 13 00:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.153,"15min":1.0569999999999999},"min":{"1min":0.73499999999999999,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.655,"5min":13.815,"15min":13.815},"last":1.1719999999999999},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.1859999999999999,"15min":1.081},"min":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.468,"5min":13.696999999999999,"15min":13.696999999999999},"last":1.083}]},{"osd":5,"last update":"Fri Sep 13 00:53:43 2024","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.323,"15min":1.1799999999999999},"min":{"1min":0.79000000000000004,"5min":0.67600000000000005,"15min":0.47899999999999998},"max":{"1min":1.3979999999999999,"5min":24.376999999999999,"15min":24.376999999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.24,"15min":1.1419999999999999},"min":{"1min":0.80800000000000005,"5min":0.752,"15min":0.57699999999999996},"max":{"1min":1.536,"5min":14.015000000000001,"15min":19.343},"last":1.1259999999999999}]},{"osd":6,"last update":"Fri Sep 13 00:53:43 2024","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.4179999999999999,"15min":1.2030000000000001},"min":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.57699999999999996},"max":{"1min":1.589,"5min":14.15,"15min":14.15},"last":0.90500000000000003},{"interface":"front","average":{"1min":1.119,"5min":1.141,"15min":1.095},"min":{"1min":0.92900000000000005,"5min":0.86399999999999999,"15min":0.56000000000000005},"max":{"1min":1.659,"5min":2.3719999999999999,"15min":8.7870000000000008},"last":1.0329999999999999}]},{"osd":7,"last update":"Fri Sep 13 00:53:43 2024","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.171,"15min":1.133},"min":{"1min":0.87,"5min":0.80500000000000005,"15min":0.58899999999999997},"max":{"1min":1.758,"5min":3.6139999999999999,"15min":9.7620000000000005},"last":0.86599999999999999},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.452,"15min":1.3180000000000001},"min":{"1min":0.93300000000000005,"5min":0.66900000000000004,"15min":0.60499999999999998},"max":{"1min":1.5009999999999999,"5min":24.452000000000002,"15min":28.82},"last":0.93899999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2404,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3500,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T00:54:15.732 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-13T00:54:16.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:16 smithi088 bash[68960]: audit 2024-09-13T00:54:14.874162+0000 mgr.y (mgr.25304) 551 : audit [DBG] from='client.44711 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:54:16.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:16 smithi088 bash[68960]: cluster 2024-09-13T00:54:14.903495+0000 mgr.y (mgr.25304) 552 : cluster [DBG] pgmap v1127: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:16.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:16 smithi077 bash[83127]: audit 2024-09-13T00:54:14.874162+0000 mgr.y (mgr.25304) 551 : audit [DBG] from='client.44711 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:54:16.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:16 smithi077 bash[83127]: cluster 2024-09-13T00:54:14.903495+0000 mgr.y (mgr.25304) 552 : cluster [DBG] pgmap v1127: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:16.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:16 smithi077 bash[80681]: audit 2024-09-13T00:54:14.874162+0000 mgr.y (mgr.25304) 551 : audit [DBG] from='client.44711 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:54:16.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:16 smithi077 bash[80681]: cluster 2024-09-13T00:54:14.903495+0000 mgr.y (mgr.25304) 552 : cluster [DBG] pgmap v1127: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:18.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:18 smithi088 bash[68960]: cluster 2024-09-13T00:54:16.904260+0000 mgr.y (mgr.25304) 553 : cluster [DBG] pgmap v1128: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:18.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:18 smithi088 bash[68960]: audit 2024-09-13T00:54:17.271214+0000 mon.a (mon.0) 1524 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:54:18.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:18 smithi077 bash[80681]: cluster 2024-09-13T00:54:16.904260+0000 mgr.y (mgr.25304) 553 : cluster [DBG] pgmap v1128: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:18.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:18 smithi077 bash[80681]: audit 2024-09-13T00:54:17.271214+0000 mon.a (mon.0) 1524 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:54:18.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:18 smithi077 bash[83127]: cluster 2024-09-13T00:54:16.904260+0000 mgr.y (mgr.25304) 553 : cluster [DBG] pgmap v1128: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:18.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:18 smithi077 bash[83127]: audit 2024-09-13T00:54:17.271214+0000 mon.a (mon.0) 1524 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:54:18.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:54:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:54:18] "GET /metrics HTTP/1.1" 200 34267 "" "Prometheus/2.43.0" 2024-09-13T00:54:20.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:20 smithi088 bash[68960]: cluster 2024-09-13T00:54:18.905088+0000 mgr.y (mgr.25304) 554 : cluster [DBG] pgmap v1129: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:20.511 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:54:20.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:20 smithi077 bash[80681]: cluster 2024-09-13T00:54:18.905088+0000 mgr.y (mgr.25304) 554 : cluster [DBG] pgmap v1129: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:20.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:20 smithi077 bash[83127]: cluster 2024-09-13T00:54:18.905088+0000 mgr.y (mgr.25304) 554 : cluster [DBG] pgmap v1129: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:22.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:22 smithi077 bash[83127]: cluster 2024-09-13T00:54:20.906020+0000 mgr.y (mgr.25304) 555 : cluster [DBG] pgmap v1130: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:22.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:22 smithi077 bash[80681]: cluster 2024-09-13T00:54:20.906020+0000 mgr.y (mgr.25304) 555 : cluster [DBG] pgmap v1130: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:22.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:22 smithi088 bash[68960]: cluster 2024-09-13T00:54:20.906020+0000 mgr.y (mgr.25304) 555 : cluster [DBG] pgmap v1130: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:22.353 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:54:22 smithi088 bash[61791]: ts=2024-09-13T00:54:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:54:23.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:23 smithi077 bash[80681]: audit 2024-09-13T00:54:22.197699+0000 mon.a (mon.0) 1525 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:54:23.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:23 smithi077 bash[80681]: audit 2024-09-13T00:54:22.204167+0000 mon.a (mon.0) 1526 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:54:23.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:23 smithi077 bash[80681]: audit 2024-09-13T00:54:23.100492+0000 mon.a (mon.0) 1527 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:54:23.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:23 smithi077 bash[80681]: audit 2024-09-13T00:54:23.112865+0000 mon.a (mon.0) 1528 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:54:23.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:23 smithi077 bash[83127]: audit 2024-09-13T00:54:22.197699+0000 mon.a (mon.0) 1525 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:54:23.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:23 smithi077 bash[83127]: audit 2024-09-13T00:54:22.204167+0000 mon.a (mon.0) 1526 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:54:23.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:23 smithi077 bash[83127]: audit 2024-09-13T00:54:23.100492+0000 mon.a (mon.0) 1527 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:54:23.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:23 smithi077 bash[83127]: audit 2024-09-13T00:54:23.112865+0000 mon.a (mon.0) 1528 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:54:23.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:23 smithi088 bash[68960]: audit 2024-09-13T00:54:22.197699+0000 mon.a (mon.0) 1525 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:54:23.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:23 smithi088 bash[68960]: audit 2024-09-13T00:54:22.204167+0000 mon.a (mon.0) 1526 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:54:23.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:23 smithi088 bash[68960]: audit 2024-09-13T00:54:23.100492+0000 mon.a (mon.0) 1527 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:54:23.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:23 smithi088 bash[68960]: audit 2024-09-13T00:54:23.112865+0000 mon.a (mon.0) 1528 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:54:24.210 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:24 smithi077 bash[80681]: cluster 2024-09-13T00:54:22.906893+0000 mgr.y (mgr.25304) 556 : cluster [DBG] pgmap v1131: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:24.210 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:24 smithi077 bash[80681]: audit 2024-09-13T00:54:23.591056+0000 mon.a (mon.0) 1529 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:54:24.210 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:24 smithi077 bash[80681]: audit 2024-09-13T00:54:23.592838+0000 mon.a (mon.0) 1530 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:54:24.210 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:24 smithi077 bash[80681]: audit 2024-09-13T00:54:23.604879+0000 mon.a (mon.0) 1531 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:54:24.282 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:54:24.284 INFO:teuthology.orchestra.run.smithi077.stderr:dumped all 2024-09-13T00:54:24.522 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:24 smithi077 bash[83127]: cluster 2024-09-13T00:54:22.906893+0000 mgr.y (mgr.25304) 556 : cluster [DBG] pgmap v1131: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:24.522 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:24 smithi077 bash[83127]: audit 2024-09-13T00:54:23.591056+0000 mon.a (mon.0) 1529 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:54:24.522 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:24 smithi077 bash[83127]: audit 2024-09-13T00:54:23.592838+0000 mon.a (mon.0) 1530 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:54:24.522 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:24 smithi077 bash[83127]: audit 2024-09-13T00:54:23.604879+0000 mon.a (mon.0) 1531 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:54:24.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:24 smithi088 bash[68960]: cluster 2024-09-13T00:54:22.906893+0000 mgr.y (mgr.25304) 556 : cluster [DBG] pgmap v1131: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:24.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:24 smithi088 bash[68960]: audit 2024-09-13T00:54:23.591056+0000 mon.a (mon.0) 1529 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:54:24.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:24 smithi088 bash[68960]: audit 2024-09-13T00:54:23.592838+0000 mon.a (mon.0) 1530 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:54:24.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:24 smithi088 bash[68960]: audit 2024-09-13T00:54:23.604879+0000 mon.a (mon.0) 1531 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:54:25.163 INFO:teuthology.orchestra.run.smithi077.stdout:{"pg_ready":true,"pg_map":{"version":1131,"stamp":"2024-09-13T00:54:22.906357+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595923,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":452,"num_read_kb":889,"num_write":237,"num_write_kb":1660,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":177,"ondisk_log_size":177,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1454196,"kb_used_data":125500,"kb_used_omap":327,"kb_used_meta":1178168,"kb_avail":748507020,"statfs":{"total":767960285184,"available":766471188480,"internally_reserved":0,"allocated":128512000,"data_stored":81080198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":335765,"internal_metadata":1206444139},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004860"},"pg_stats":[{"pgid":"3.1f","version":"63'5","reported_seq":911,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.327228+0000","last_change":"2024-09-13T00:45:41.626626+0000","last_active":"2024-09-13T00:52:40.327228+0000","last_peered":"2024-09-13T00:52:40.327228+0000","last_clean":"2024-09-13T00:52:40.327228+0000","last_became_active":"2024-09-13T00:45:41.626278+0000","last_became_peered":"2024-09-13T00:45:41.626278+0000","last_unstale":"2024-09-13T00:52:40.327228+0000","last_undegraded":"2024-09-13T00:52:40.327228+0000","last_fullsized":"2024-09-13T00:52:40.327228+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:42:58.455220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":923,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.258186+0000","last_change":"2024-09-13T00:46:45.517960+0000","last_active":"2024-09-13T00:52:40.258186+0000","last_peered":"2024-09-13T00:52:40.258186+0000","last_clean":"2024-09-13T00:52:40.258186+0000","last_became_active":"2024-09-13T00:46:45.517434+0000","last_became_peered":"2024-09-13T00:46:45.517434+0000","last_unstale":"2024-09-13T00:52:40.258186+0000","last_undegraded":"2024-09-13T00:52:40.258186+0000","last_fullsized":"2024-09-13T00:52:40.258186+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:40:06.681450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"63'2","reported_seq":920,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.320597+0000","last_change":"2024-09-13T00:46:15.330699+0000","last_active":"2024-09-13T00:52:40.320597+0000","last_peered":"2024-09-13T00:52:40.320597+0000","last_clean":"2024-09-13T00:52:40.320597+0000","last_became_active":"2024-09-13T00:46:15.325486+0000","last_became_peered":"2024-09-13T00:46:15.325486+0000","last_unstale":"2024-09-13T00:52:40.320597+0000","last_undegraded":"2024-09-13T00:52:40.320597+0000","last_fullsized":"2024-09-13T00:52:40.320597+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:13:40.129831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":940,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.321114+0000","last_change":"2024-09-13T00:46:15.321514+0000","last_active":"2024-09-13T00:52:40.321114+0000","last_peered":"2024-09-13T00:52:40.321114+0000","last_clean":"2024-09-13T00:52:40.321114+0000","last_became_active":"2024-09-13T00:46:15.320037+0000","last_became_peered":"2024-09-13T00:46:15.320037+0000","last_unstale":"2024-09-13T00:52:40.321114+0000","last_undegraded":"2024-09-13T00:52:40.321114+0000","last_fullsized":"2024-09-13T00:52:40.321114+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:40:56.856055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"62'2","reported_seq":913,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.259414+0000","last_change":"2024-09-13T00:46:45.507901+0000","last_active":"2024-09-13T00:52:39.259414+0000","last_peered":"2024-09-13T00:52:39.259414+0000","last_clean":"2024-09-13T00:52:39.259414+0000","last_became_active":"2024-09-13T00:46:45.507081+0000","last_became_peered":"2024-09-13T00:46:45.507081+0000","last_unstale":"2024-09-13T00:52:39.259414+0000","last_undegraded":"2024-09-13T00:52:39.259414+0000","last_fullsized":"2024-09-13T00:52:39.259414+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:12:06.783019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":944,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.270348+0000","last_change":"2024-09-13T00:46:45.509213+0000","last_active":"2024-09-13T00:52:40.270348+0000","last_peered":"2024-09-13T00:52:40.270348+0000","last_clean":"2024-09-13T00:52:40.270348+0000","last_became_active":"2024-09-13T00:46:45.508648+0000","last_became_peered":"2024-09-13T00:46:45.508648+0000","last_unstale":"2024-09-13T00:52:40.270348+0000","last_undegraded":"2024-09-13T00:52:40.270348+0000","last_fullsized":"2024-09-13T00:52:40.270348+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:21:41.245775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"62'2","reported_seq":930,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.258592+0000","last_change":"2024-09-13T00:46:45.522136+0000","last_active":"2024-09-13T00:52:39.258592+0000","last_peered":"2024-09-13T00:52:39.258592+0000","last_clean":"2024-09-13T00:52:39.258592+0000","last_became_active":"2024-09-13T00:46:45.521754+0000","last_became_peered":"2024-09-13T00:46:45.521754+0000","last_unstale":"2024-09-13T00:52:39.258592+0000","last_undegraded":"2024-09-13T00:52:39.258592+0000","last_fullsized":"2024-09-13T00:52:39.258592+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:31:59.251880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":944,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.261104+0000","last_change":"2024-09-13T00:46:45.535236+0000","last_active":"2024-09-13T00:52:40.261104+0000","last_peered":"2024-09-13T00:52:40.261104+0000","last_clean":"2024-09-13T00:52:40.261104+0000","last_became_active":"2024-09-13T00:46:45.534956+0000","last_became_peered":"2024-09-13T00:46:45.534956+0000","last_unstale":"2024-09-13T00:52:40.261104+0000","last_undegraded":"2024-09-13T00:52:40.261104+0000","last_fullsized":"2024-09-13T00:52:40.261104+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:55:47.169063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"63'2","reported_seq":905,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.323022+0000","last_change":"2024-09-13T00:46:45.518960+0000","last_active":"2024-09-13T00:52:40.323022+0000","last_peered":"2024-09-13T00:52:40.323022+0000","last_clean":"2024-09-13T00:52:40.323022+0000","last_became_active":"2024-09-13T00:46:45.517233+0000","last_became_peered":"2024-09-13T00:46:45.517233+0000","last_unstale":"2024-09-13T00:52:40.323022+0000","last_undegraded":"2024-09-13T00:52:40.323022+0000","last_fullsized":"2024-09-13T00:52:40.323022+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:33:04.004456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":941,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.259178+0000","last_change":"2024-09-13T00:46:15.344315+0000","last_active":"2024-09-13T00:52:40.259178+0000","last_peered":"2024-09-13T00:52:40.259178+0000","last_clean":"2024-09-13T00:52:40.259178+0000","last_became_active":"2024-09-13T00:46:15.343998+0000","last_became_peered":"2024-09-13T00:46:15.343998+0000","last_unstale":"2024-09-13T00:52:40.259178+0000","last_undegraded":"2024-09-13T00:52:40.259178+0000","last_fullsized":"2024-09-13T00:52:40.259178+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T12:20:50.373554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"63'2","reported_seq":933,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.260215+0000","last_change":"2024-09-13T00:46:45.529526+0000","last_active":"2024-09-13T00:52:40.260215+0000","last_peered":"2024-09-13T00:52:40.260215+0000","last_clean":"2024-09-13T00:52:40.260215+0000","last_became_active":"2024-09-13T00:46:45.527232+0000","last_became_peered":"2024-09-13T00:46:45.527232+0000","last_unstale":"2024-09-13T00:52:40.260215+0000","last_undegraded":"2024-09-13T00:52:40.260215+0000","last_fullsized":"2024-09-13T00:52:40.260215+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:44:35.336089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":938,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.319032+0000","last_change":"2024-09-13T00:46:45.522928+0000","last_active":"2024-09-13T00:52:40.319032+0000","last_peered":"2024-09-13T00:52:40.319032+0000","last_clean":"2024-09-13T00:52:40.319032+0000","last_became_active":"2024-09-13T00:46:45.521955+0000","last_became_peered":"2024-09-13T00:46:45.521955+0000","last_unstale":"2024-09-13T00:52:40.319032+0000","last_undegraded":"2024-09-13T00:52:40.319032+0000","last_fullsized":"2024-09-13T00:52:40.319032+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:01:30.288065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":918,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.324428+0000","last_change":"2024-09-13T00:46:45.530887+0000","last_active":"2024-09-13T00:52:40.324428+0000","last_peered":"2024-09-13T00:52:40.324428+0000","last_clean":"2024-09-13T00:52:40.324428+0000","last_became_active":"2024-09-13T00:46:45.530400+0000","last_became_peered":"2024-09-13T00:46:45.530400+0000","last_unstale":"2024-09-13T00:52:40.324428+0000","last_undegraded":"2024-09-13T00:52:40.324428+0000","last_fullsized":"2024-09-13T00:52:40.324428+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:31:55.473617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":901,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.319151+0000","last_change":"2024-09-13T00:46:15.331478+0000","last_active":"2024-09-13T00:52:40.319151+0000","last_peered":"2024-09-13T00:52:40.319151+0000","last_clean":"2024-09-13T00:52:40.319151+0000","last_became_active":"2024-09-13T00:46:15.331202+0000","last_became_peered":"2024-09-13T00:46:15.331202+0000","last_unstale":"2024-09-13T00:52:40.319151+0000","last_undegraded":"2024-09-13T00:52:40.319151+0000","last_fullsized":"2024-09-13T00:52:40.319151+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:35:28.914097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":941,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.313324+0000","last_change":"2024-09-13T00:46:15.341826+0000","last_active":"2024-09-13T00:52:40.313324+0000","last_peered":"2024-09-13T00:52:40.313324+0000","last_clean":"2024-09-13T00:52:40.313324+0000","last_became_active":"2024-09-13T00:46:15.341132+0000","last_became_peered":"2024-09-13T00:46:15.341132+0000","last_unstale":"2024-09-13T00:52:40.313324+0000","last_undegraded":"2024-09-13T00:52:40.313324+0000","last_fullsized":"2024-09-13T00:52:40.313324+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:37:09.570985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":915,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.432015+0000","last_change":"2024-09-13T00:43:56.096526+0000","last_active":"2024-09-13T00:52:39.432015+0000","last_peered":"2024-09-13T00:52:39.432015+0000","last_clean":"2024-09-13T00:52:39.432015+0000","last_became_active":"2024-09-13T00:43:56.094929+0000","last_became_peered":"2024-09-13T00:43:56.094929+0000","last_unstale":"2024-09-13T00:52:39.432015+0000","last_undegraded":"2024-09-13T00:52:39.432015+0000","last_fullsized":"2024-09-13T00:52:39.432015+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:24:56.143144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":923,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.313961+0000","last_change":"2024-09-13T00:46:15.339591+0000","last_active":"2024-09-13T00:52:40.313961+0000","last_peered":"2024-09-13T00:52:40.313961+0000","last_clean":"2024-09-13T00:52:40.313961+0000","last_became_active":"2024-09-13T00:46:15.338782+0000","last_became_peered":"2024-09-13T00:46:15.338782+0000","last_unstale":"2024-09-13T00:52:40.313961+0000","last_undegraded":"2024-09-13T00:52:40.313961+0000","last_fullsized":"2024-09-13T00:52:40.313961+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:11:30.414422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":900,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.326970+0000","last_change":"2024-09-13T00:46:15.328788+0000","last_active":"2024-09-13T00:52:40.326970+0000","last_peered":"2024-09-13T00:52:40.326970+0000","last_clean":"2024-09-13T00:52:40.326970+0000","last_became_active":"2024-09-13T00:46:15.328123+0000","last_became_peered":"2024-09-13T00:46:15.328123+0000","last_unstale":"2024-09-13T00:52:40.326970+0000","last_undegraded":"2024-09-13T00:52:40.326970+0000","last_fullsized":"2024-09-13T00:52:40.326970+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:47:38.824723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":906,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.324737+0000","last_change":"2024-09-13T00:46:15.333000+0000","last_active":"2024-09-13T00:52:40.324737+0000","last_peered":"2024-09-13T00:52:40.324737+0000","last_clean":"2024-09-13T00:52:40.324737+0000","last_became_active":"2024-09-13T00:46:15.332581+0000","last_became_peered":"2024-09-13T00:46:15.332581+0000","last_unstale":"2024-09-13T00:52:40.324737+0000","last_undegraded":"2024-09-13T00:52:40.324737+0000","last_fullsized":"2024-09-13T00:52:40.324737+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:01:06.004637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":910,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.259738+0000","last_change":"2024-09-13T00:43:56.097452+0000","last_active":"2024-09-13T00:52:39.259738+0000","last_peered":"2024-09-13T00:52:39.259738+0000","last_clean":"2024-09-13T00:52:39.259738+0000","last_became_active":"2024-09-13T00:43:56.096578+0000","last_became_peered":"2024-09-13T00:43:56.096578+0000","last_unstale":"2024-09-13T00:52:39.259738+0000","last_undegraded":"2024-09-13T00:52:39.259738+0000","last_fullsized":"2024-09-13T00:52:39.259738+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:59:21.650017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"62'3","reported_seq":935,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.279968+0000","last_change":"2024-09-13T00:46:45.503242+0000","last_active":"2024-09-13T00:52:39.279968+0000","last_peered":"2024-09-13T00:52:39.279968+0000","last_clean":"2024-09-13T00:52:39.279968+0000","last_became_active":"2024-09-13T00:46:45.502312+0000","last_became_peered":"2024-09-13T00:46:45.502312+0000","last_unstale":"2024-09-13T00:52:39.279968+0000","last_undegraded":"2024-09-13T00:52:39.279968+0000","last_fullsized":"2024-09-13T00:52:39.279968+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:20:37.698979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":903,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.317441+0000","last_change":"2024-09-13T00:45:41.598285+0000","last_active":"2024-09-13T00:52:40.317441+0000","last_peered":"2024-09-13T00:52:40.317441+0000","last_clean":"2024-09-13T00:52:40.317441+0000","last_became_active":"2024-09-13T00:45:41.597845+0000","last_became_peered":"2024-09-13T00:45:41.597845+0000","last_unstale":"2024-09-13T00:52:40.317441+0000","last_undegraded":"2024-09-13T00:52:40.317441+0000","last_fullsized":"2024-09-13T00:52:40.317441+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:58:24.168311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"368'24","reported_seq":911,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.260827+0000","last_change":"2024-09-13T00:46:45.493995+0000","last_active":"2024-09-13T00:52:40.260827+0000","last_peered":"2024-09-13T00:52:40.260827+0000","last_clean":"2024-09-13T00:52:40.260827+0000","last_became_active":"2024-09-13T00:46:45.492736+0000","last_became_peered":"2024-09-13T00:46:45.492736+0000","last_unstale":"2024-09-13T00:52:40.260827+0000","last_undegraded":"2024-09-13T00:52:40.260827+0000","last_fullsized":"2024-09-13T00:52:40.260827+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":24,"log_dups_size":0,"ondisk_log_size":24,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:12:06.114096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4750,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":7,"num_read_kb":6,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":906,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.260843+0000","last_change":"2024-09-13T00:46:45.529941+0000","last_active":"2024-09-13T00:52:40.260843+0000","last_peered":"2024-09-13T00:52:40.260843+0000","last_clean":"2024-09-13T00:52:40.260843+0000","last_became_active":"2024-09-13T00:46:45.529071+0000","last_became_peered":"2024-09-13T00:46:45.529071+0000","last_unstale":"2024-09-13T00:52:40.260843+0000","last_undegraded":"2024-09-13T00:52:40.260843+0000","last_fullsized":"2024-09-13T00:52:40.260843+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:54:30.695569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":902,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.262323+0000","last_change":"2024-09-13T00:46:45.515148+0000","last_active":"2024-09-13T00:52:40.262323+0000","last_peered":"2024-09-13T00:52:40.262323+0000","last_clean":"2024-09-13T00:52:40.262323+0000","last_became_active":"2024-09-13T00:46:45.500689+0000","last_became_peered":"2024-09-13T00:46:45.500689+0000","last_unstale":"2024-09-13T00:52:40.262323+0000","last_undegraded":"2024-09-13T00:52:40.262323+0000","last_fullsized":"2024-09-13T00:52:40.262323+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:22:50.405443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":928,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.432161+0000","last_change":"2024-09-13T00:45:41.629188+0000","last_active":"2024-09-13T00:52:39.432161+0000","last_peered":"2024-09-13T00:52:39.432161+0000","last_clean":"2024-09-13T00:52:39.432161+0000","last_became_active":"2024-09-13T00:45:41.626966+0000","last_became_peered":"2024-09-13T00:45:41.626966+0000","last_unstale":"2024-09-13T00:52:39.432161+0000","last_undegraded":"2024-09-13T00:52:39.432161+0000","last_fullsized":"2024-09-13T00:52:39.432161+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:06:08.792854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00135603,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":920,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.318851+0000","last_change":"2024-09-13T00:46:45.519936+0000","last_active":"2024-09-13T00:52:40.318851+0000","last_peered":"2024-09-13T00:52:40.318851+0000","last_clean":"2024-09-13T00:52:40.318851+0000","last_became_active":"2024-09-13T00:46:45.519473+0000","last_became_peered":"2024-09-13T00:46:45.519473+0000","last_unstale":"2024-09-13T00:52:40.318851+0000","last_undegraded":"2024-09-13T00:52:40.318851+0000","last_fullsized":"2024-09-13T00:52:40.318851+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:26:37.351469+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":924,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.313542+0000","last_change":"2024-09-13T00:46:15.334557+0000","last_active":"2024-09-13T00:52:40.313542+0000","last_peered":"2024-09-13T00:52:40.313542+0000","last_clean":"2024-09-13T00:52:40.313542+0000","last_became_active":"2024-09-13T00:46:15.333534+0000","last_became_peered":"2024-09-13T00:46:15.333534+0000","last_unstale":"2024-09-13T00:52:40.313542+0000","last_undegraded":"2024-09-13T00:52:40.313542+0000","last_fullsized":"2024-09-13T00:52:40.313542+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:39:10.101248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":913,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.272636+0000","last_change":"2024-09-13T00:46:45.529937+0000","last_active":"2024-09-13T00:52:39.272636+0000","last_peered":"2024-09-13T00:52:39.272636+0000","last_clean":"2024-09-13T00:52:39.272636+0000","last_became_active":"2024-09-13T00:46:45.529259+0000","last_became_peered":"2024-09-13T00:46:45.529259+0000","last_unstale":"2024-09-13T00:52:39.272636+0000","last_undegraded":"2024-09-13T00:52:39.272636+0000","last_fullsized":"2024-09-13T00:52:39.272636+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:56:18.590692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"62'2","reported_seq":927,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.320734+0000","last_change":"2024-09-13T00:46:15.335239+0000","last_active":"2024-09-13T00:52:40.320734+0000","last_peered":"2024-09-13T00:52:40.320734+0000","last_clean":"2024-09-13T00:52:40.320734+0000","last_became_active":"2024-09-13T00:46:15.334885+0000","last_became_peered":"2024-09-13T00:46:15.334885+0000","last_unstale":"2024-09-13T00:52:40.320734+0000","last_undegraded":"2024-09-13T00:52:40.320734+0000","last_fullsized":"2024-09-13T00:52:40.320734+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:43:15.769703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":886,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.323382+0000","last_change":"2024-09-13T00:43:18.180239+0000","last_active":"2024-09-13T00:52:40.323382+0000","last_peered":"2024-09-13T00:52:40.323382+0000","last_clean":"2024-09-13T00:52:40.323382+0000","last_became_active":"2024-09-13T00:43:18.179413+0000","last_became_peered":"2024-09-13T00:43:18.179413+0000","last_unstale":"2024-09-13T00:52:40.323382+0000","last_undegraded":"2024-09-13T00:52:40.323382+0000","last_fullsized":"2024-09-13T00:52:40.323382+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:18:03.116471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":942,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.280376+0000","last_change":"2024-09-13T00:46:45.503256+0000","last_active":"2024-09-13T00:52:39.280376+0000","last_peered":"2024-09-13T00:52:39.280376+0000","last_clean":"2024-09-13T00:52:39.280376+0000","last_became_active":"2024-09-13T00:46:45.502770+0000","last_became_peered":"2024-09-13T00:46:45.502770+0000","last_unstale":"2024-09-13T00:52:39.280376+0000","last_undegraded":"2024-09-13T00:52:39.280376+0000","last_fullsized":"2024-09-13T00:52:39.280376+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:07:22.957742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077602300000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":904,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.322084+0000","last_change":"2024-09-13T00:45:41.601473+0000","last_active":"2024-09-13T00:52:40.322084+0000","last_peered":"2024-09-13T00:52:40.322084+0000","last_clean":"2024-09-13T00:52:40.322084+0000","last_became_active":"2024-09-13T00:45:41.600376+0000","last_became_peered":"2024-09-13T00:45:41.600376+0000","last_unstale":"2024-09-13T00:52:40.322084+0000","last_undegraded":"2024-09-13T00:52:40.322084+0000","last_fullsized":"2024-09-13T00:52:40.322084+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:37:33.287548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":904,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.321262+0000","last_change":"2024-09-13T00:46:45.532330+0000","last_active":"2024-09-13T00:52:40.321262+0000","last_peered":"2024-09-13T00:52:40.321262+0000","last_clean":"2024-09-13T00:52:40.321262+0000","last_became_active":"2024-09-13T00:46:45.531246+0000","last_became_peered":"2024-09-13T00:46:45.531246+0000","last_unstale":"2024-09-13T00:52:40.321262+0000","last_undegraded":"2024-09-13T00:52:40.321262+0000","last_fullsized":"2024-09-13T00:52:40.321262+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:49:33.378939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":909,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.321844+0000","last_change":"2024-09-13T00:43:56.101142+0000","last_active":"2024-09-13T00:52:40.321844+0000","last_peered":"2024-09-13T00:52:40.321844+0000","last_clean":"2024-09-13T00:52:40.321844+0000","last_became_active":"2024-09-13T00:43:56.098893+0000","last_became_peered":"2024-09-13T00:43:56.098893+0000","last_unstale":"2024-09-13T00:52:40.321844+0000","last_undegraded":"2024-09-13T00:52:40.321844+0000","last_fullsized":"2024-09-13T00:52:40.321844+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:04:44.962433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053219400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":927,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.259613+0000","last_change":"2024-09-13T00:46:15.327798+0000","last_active":"2024-09-13T00:52:39.259613+0000","last_peered":"2024-09-13T00:52:39.259613+0000","last_clean":"2024-09-13T00:52:39.259613+0000","last_became_active":"2024-09-13T00:46:15.327504+0000","last_became_peered":"2024-09-13T00:46:15.327504+0000","last_unstale":"2024-09-13T00:52:39.259613+0000","last_undegraded":"2024-09-13T00:52:39.259613+0000","last_fullsized":"2024-09-13T00:52:39.259613+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:02:29.779493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":919,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.322164+0000","last_change":"2024-09-13T00:46:15.354292+0000","last_active":"2024-09-13T00:52:40.322164+0000","last_peered":"2024-09-13T00:52:40.322164+0000","last_clean":"2024-09-13T00:52:40.322164+0000","last_became_active":"2024-09-13T00:46:15.352070+0000","last_became_peered":"2024-09-13T00:46:15.352070+0000","last_unstale":"2024-09-13T00:52:40.322164+0000","last_undegraded":"2024-09-13T00:52:40.322164+0000","last_fullsized":"2024-09-13T00:52:40.322164+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:14:17.847476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":911,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.260725+0000","last_change":"2024-09-13T00:46:45.517569+0000","last_active":"2024-09-13T00:52:40.260725+0000","last_peered":"2024-09-13T00:52:40.260725+0000","last_clean":"2024-09-13T00:52:40.260725+0000","last_became_active":"2024-09-13T00:46:45.516270+0000","last_became_peered":"2024-09-13T00:46:45.516270+0000","last_unstale":"2024-09-13T00:52:40.260725+0000","last_undegraded":"2024-09-13T00:52:40.260725+0000","last_fullsized":"2024-09-13T00:52:40.260725+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:46:41.924008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"57'2","reported_seq":924,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.258313+0000","last_change":"2024-09-13T00:43:56.096241+0000","last_active":"2024-09-13T00:52:39.258313+0000","last_peered":"2024-09-13T00:52:39.258313+0000","last_clean":"2024-09-13T00:52:39.258313+0000","last_became_active":"2024-09-13T00:43:56.095239+0000","last_became_peered":"2024-09-13T00:43:56.095239+0000","last_unstale":"2024-09-13T00:52:39.258313+0000","last_undegraded":"2024-09-13T00:52:39.258313+0000","last_fullsized":"2024-09-13T00:52:39.258313+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:58:15.629030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.002059707,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":895,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.272286+0000","last_change":"2024-09-13T00:43:56.114793+0000","last_active":"2024-09-13T00:52:39.272286+0000","last_peered":"2024-09-13T00:52:39.272286+0000","last_clean":"2024-09-13T00:52:39.272286+0000","last_became_active":"2024-09-13T00:43:56.109668+0000","last_became_peered":"2024-09-13T00:43:56.109668+0000","last_unstale":"2024-09-13T00:52:39.272286+0000","last_undegraded":"2024-09-13T00:52:39.272286+0000","last_fullsized":"2024-09-13T00:52:39.272286+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:32:06.789192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":904,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.322248+0000","last_change":"2024-09-13T00:45:41.597941+0000","last_active":"2024-09-13T00:52:40.322248+0000","last_peered":"2024-09-13T00:52:40.322248+0000","last_clean":"2024-09-13T00:52:40.322248+0000","last_became_active":"2024-09-13T00:45:41.597499+0000","last_became_peered":"2024-09-13T00:45:41.597499+0000","last_unstale":"2024-09-13T00:52:40.322248+0000","last_undegraded":"2024-09-13T00:52:40.322248+0000","last_fullsized":"2024-09-13T00:52:40.322248+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:56:08.198126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":919,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.258023+0000","last_change":"2024-09-13T00:46:15.312908+0000","last_active":"2024-09-13T00:52:40.258023+0000","last_peered":"2024-09-13T00:52:40.258023+0000","last_clean":"2024-09-13T00:52:40.258023+0000","last_became_active":"2024-09-13T00:46:15.312536+0000","last_became_peered":"2024-09-13T00:46:15.312536+0000","last_unstale":"2024-09-13T00:52:40.258023+0000","last_undegraded":"2024-09-13T00:52:40.258023+0000","last_fullsized":"2024-09-13T00:52:40.258023+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:57:23.320972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00067555600000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":916,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.319294+0000","last_change":"2024-09-13T00:45:41.619166+0000","last_active":"2024-09-13T00:52:40.319294+0000","last_peered":"2024-09-13T00:52:40.319294+0000","last_clean":"2024-09-13T00:52:40.319294+0000","last_became_active":"2024-09-13T00:45:41.618844+0000","last_became_peered":"2024-09-13T00:45:41.618844+0000","last_unstale":"2024-09-13T00:52:40.319294+0000","last_undegraded":"2024-09-13T00:52:40.319294+0000","last_fullsized":"2024-09-13T00:52:40.319294+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:02:14.691566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":916,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.330943+0000","last_change":"2024-09-13T00:46:45.496190+0000","last_active":"2024-09-13T00:52:40.330943+0000","last_peered":"2024-09-13T00:52:40.330943+0000","last_clean":"2024-09-13T00:52:40.330943+0000","last_became_active":"2024-09-13T00:46:45.495623+0000","last_became_peered":"2024-09-13T00:46:45.495623+0000","last_unstale":"2024-09-13T00:52:40.330943+0000","last_undegraded":"2024-09-13T00:52:40.330943+0000","last_fullsized":"2024-09-13T00:52:40.330943+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:33:20.680033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":942,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.280291+0000","last_change":"2024-09-13T00:43:56.094126+0000","last_active":"2024-09-13T00:52:39.280291+0000","last_peered":"2024-09-13T00:52:39.280291+0000","last_clean":"2024-09-13T00:52:39.280291+0000","last_became_active":"2024-09-13T00:43:56.093290+0000","last_became_peered":"2024-09-13T00:43:56.093290+0000","last_unstale":"2024-09-13T00:52:39.280291+0000","last_undegraded":"2024-09-13T00:52:39.280291+0000","last_fullsized":"2024-09-13T00:52:39.280291+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:26:40.546096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00062469399999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":915,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.432380+0000","last_change":"2024-09-13T00:46:45.486085+0000","last_active":"2024-09-13T00:52:39.432380+0000","last_peered":"2024-09-13T00:52:39.432380+0000","last_clean":"2024-09-13T00:52:39.432380+0000","last_became_active":"2024-09-13T00:46:45.485596+0000","last_became_peered":"2024-09-13T00:46:45.485596+0000","last_unstale":"2024-09-13T00:52:39.432380+0000","last_undegraded":"2024-09-13T00:52:39.432380+0000","last_fullsized":"2024-09-13T00:52:39.432380+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:54:51.909304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":905,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.264936+0000","last_change":"2024-09-13T00:46:15.337295+0000","last_active":"2024-09-13T00:52:39.264936+0000","last_peered":"2024-09-13T00:52:39.264936+0000","last_clean":"2024-09-13T00:52:39.264936+0000","last_became_active":"2024-09-13T00:46:15.336890+0000","last_became_peered":"2024-09-13T00:46:15.336890+0000","last_unstale":"2024-09-13T00:52:39.264936+0000","last_undegraded":"2024-09-13T00:52:39.264936+0000","last_fullsized":"2024-09-13T00:52:39.264936+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T12:31:43.952771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"55'1","reported_seq":923,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.258734+0000","last_change":"2024-09-13T00:45:41.620277+0000","last_active":"2024-09-13T00:52:39.258734+0000","last_peered":"2024-09-13T00:52:39.258734+0000","last_clean":"2024-09-13T00:52:39.258734+0000","last_became_active":"2024-09-13T00:45:41.619579+0000","last_became_peered":"2024-09-13T00:45:41.619579+0000","last_unstale":"2024-09-13T00:52:39.258734+0000","last_undegraded":"2024-09-13T00:52:39.258734+0000","last_fullsized":"2024-09-13T00:52:39.258734+0000","mapping_epoch":629,"log_start":"0'0","ondisk_log_start":"0'0","created":52,"last_epoch_clean":630,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:30:09.965256+0000","last_clean_scrub_stamp":"2024-09-13T00:30:09.965256+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:10:40.119383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096918099999999995,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"292'112","reported_seq":1072,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.279491+0000","last_change":"2024-09-13T00:43:18.168213+0000","last_active":"2024-09-13T00:52:39.279491+0000","last_peered":"2024-09-13T00:52:39.279491+0000","last_clean":"2024-09-13T00:52:39.279491+0000","last_became_active":"2024-09-13T00:43:18.167226+0000","last_became_peered":"2024-09-13T00:43:18.167226+0000","last_unstale":"2024-09-13T00:52:39.279491+0000","last_undegraded":"2024-09-13T00:52:39.279491+0000","last_fullsized":"2024-09-13T00:52:39.279491+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:24:16.220413+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:24:16.220413+0000","last_clean_scrub_stamp":"2024-09-13T00:24:16.220413+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:55:23.652528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":414,"num_read_kb":849,"num_write":173,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":918,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.321743+0000","last_change":"2024-09-13T00:46:45.530576+0000","last_active":"2024-09-13T00:52:40.321743+0000","last_peered":"2024-09-13T00:52:40.321743+0000","last_clean":"2024-09-13T00:52:40.321743+0000","last_became_active":"2024-09-13T00:46:45.529881+0000","last_became_peered":"2024-09-13T00:46:45.529881+0000","last_unstale":"2024-09-13T00:52:40.321743+0000","last_undegraded":"2024-09-13T00:52:40.321743+0000","last_fullsized":"2024-09-13T00:52:40.321743+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T01:52:55.873479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":924,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.313757+0000","last_change":"2024-09-13T00:46:15.341519+0000","last_active":"2024-09-13T00:52:40.313757+0000","last_peered":"2024-09-13T00:52:40.313757+0000","last_clean":"2024-09-13T00:52:40.313757+0000","last_became_active":"2024-09-13T00:46:15.340548+0000","last_became_peered":"2024-09-13T00:46:15.340548+0000","last_unstale":"2024-09-13T00:52:40.313757+0000","last_undegraded":"2024-09-13T00:52:40.313757+0000","last_fullsized":"2024-09-13T00:52:40.313757+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:33:57.368092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":906,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.262146+0000","last_change":"2024-09-13T00:46:45.526955+0000","last_active":"2024-09-13T00:52:40.262146+0000","last_peered":"2024-09-13T00:52:40.262146+0000","last_clean":"2024-09-13T00:52:40.262146+0000","last_became_active":"2024-09-13T00:46:45.526699+0000","last_became_peered":"2024-09-13T00:46:45.526699+0000","last_unstale":"2024-09-13T00:52:40.262146+0000","last_undegraded":"2024-09-13T00:52:40.262146+0000","last_fullsized":"2024-09-13T00:52:40.262146+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:39:27.703444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":914,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.260160+0000","last_change":"2024-09-13T00:46:45.521123+0000","last_active":"2024-09-13T00:52:40.260160+0000","last_peered":"2024-09-13T00:52:40.260160+0000","last_clean":"2024-09-13T00:52:40.260160+0000","last_became_active":"2024-09-13T00:46:45.519894+0000","last_became_peered":"2024-09-13T00:46:45.519894+0000","last_unstale":"2024-09-13T00:52:40.260160+0000","last_undegraded":"2024-09-13T00:52:40.260160+0000","last_fullsized":"2024-09-13T00:52:40.260160+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:27:03.451547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":930,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.261925+0000","last_change":"2024-09-13T00:46:45.521910+0000","last_active":"2024-09-13T00:52:40.261925+0000","last_peered":"2024-09-13T00:52:40.261925+0000","last_clean":"2024-09-13T00:52:40.261925+0000","last_became_active":"2024-09-13T00:46:45.520547+0000","last_became_peered":"2024-09-13T00:46:45.520547+0000","last_unstale":"2024-09-13T00:52:40.261925+0000","last_undegraded":"2024-09-13T00:52:40.261925+0000","last_fullsized":"2024-09-13T00:52:40.261925+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:57:43.072817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":933,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.431407+0000","last_change":"2024-09-13T00:46:45.518980+0000","last_active":"2024-09-13T00:52:39.431407+0000","last_peered":"2024-09-13T00:52:39.431407+0000","last_clean":"2024-09-13T00:52:39.431407+0000","last_became_active":"2024-09-13T00:46:45.518676+0000","last_became_peered":"2024-09-13T00:46:45.518676+0000","last_unstale":"2024-09-13T00:52:39.431407+0000","last_undegraded":"2024-09-13T00:52:39.431407+0000","last_fullsized":"2024-09-13T00:52:39.431407+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:07:29.535519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":927,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.261060+0000","last_change":"2024-09-13T00:46:45.509062+0000","last_active":"2024-09-13T00:52:40.261060+0000","last_peered":"2024-09-13T00:52:40.261060+0000","last_clean":"2024-09-13T00:52:40.261060+0000","last_became_active":"2024-09-13T00:46:45.504287+0000","last_became_peered":"2024-09-13T00:46:45.504287+0000","last_unstale":"2024-09-13T00:52:40.261060+0000","last_undegraded":"2024-09-13T00:52:40.261060+0000","last_fullsized":"2024-09-13T00:52:40.261060+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T00:35:27.422911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"63'3","reported_seq":943,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.321992+0000","last_change":"2024-09-13T00:46:45.516465+0000","last_active":"2024-09-13T00:52:40.321992+0000","last_peered":"2024-09-13T00:52:40.321992+0000","last_clean":"2024-09-13T00:52:40.321992+0000","last_became_active":"2024-09-13T00:46:45.515275+0000","last_became_peered":"2024-09-13T00:46:45.515275+0000","last_unstale":"2024-09-13T00:52:40.321992+0000","last_undegraded":"2024-09-13T00:52:40.321992+0000","last_fullsized":"2024-09-13T00:52:40.321992+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:44:06.608978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":929,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.276547+0000","last_change":"2024-09-13T00:46:45.519171+0000","last_active":"2024-09-13T00:52:39.276547+0000","last_peered":"2024-09-13T00:52:39.276547+0000","last_clean":"2024-09-13T00:52:39.276547+0000","last_became_active":"2024-09-13T00:46:45.518674+0000","last_became_peered":"2024-09-13T00:46:45.518674+0000","last_unstale":"2024-09-13T00:52:39.276547+0000","last_undegraded":"2024-09-13T00:52:39.276547+0000","last_fullsized":"2024-09-13T00:52:39.276547+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T07:43:10.733090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":910,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.258570+0000","last_change":"2024-09-13T00:46:15.327439+0000","last_active":"2024-09-13T00:52:39.258570+0000","last_peered":"2024-09-13T00:52:39.258570+0000","last_clean":"2024-09-13T00:52:39.258570+0000","last_became_active":"2024-09-13T00:46:15.326691+0000","last_became_peered":"2024-09-13T00:46:15.326691+0000","last_unstale":"2024-09-13T00:52:39.258570+0000","last_undegraded":"2024-09-13T00:52:39.258570+0000","last_fullsized":"2024-09-13T00:52:39.258570+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:52:52.553904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":915,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.431162+0000","last_change":"2024-09-13T00:43:56.097526+0000","last_active":"2024-09-13T00:52:39.431162+0000","last_peered":"2024-09-13T00:52:39.431162+0000","last_clean":"2024-09-13T00:52:39.431162+0000","last_became_active":"2024-09-13T00:43:56.096340+0000","last_became_peered":"2024-09-13T00:43:56.096340+0000","last_unstale":"2024-09-13T00:52:39.431162+0000","last_undegraded":"2024-09-13T00:52:39.431162+0000","last_fullsized":"2024-09-13T00:52:39.431162+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:41:33.452908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"62'1","reported_seq":898,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.280351+0000","last_change":"2024-09-13T00:32:23.122340+0000","last_active":"2024-09-13T00:52:39.280351+0000","last_peered":"2024-09-13T00:52:39.280351+0000","last_clean":"2024-09-13T00:52:39.280351+0000","last_became_active":"2024-09-13T00:32:23.121795+0000","last_became_peered":"2024-09-13T00:32:23.121795+0000","last_unstale":"2024-09-13T00:52:39.280351+0000","last_undegraded":"2024-09-13T00:52:39.280351+0000","last_fullsized":"2024-09-13T00:52:39.280351+0000","mapping_epoch":59,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:18:28.661306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":937,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.260382+0000","last_change":"2024-09-13T00:46:45.515627+0000","last_active":"2024-09-13T00:52:40.260382+0000","last_peered":"2024-09-13T00:52:40.260382+0000","last_clean":"2024-09-13T00:52:40.260382+0000","last_became_active":"2024-09-13T00:46:45.509541+0000","last_became_peered":"2024-09-13T00:46:45.509541+0000","last_unstale":"2024-09-13T00:52:40.260382+0000","last_undegraded":"2024-09-13T00:52:40.260382+0000","last_fullsized":"2024-09-13T00:52:40.260382+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T08:46:15.698098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"62'2","reported_seq":933,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.280246+0000","last_change":"2024-09-13T00:46:45.523694+0000","last_active":"2024-09-13T00:52:39.280246+0000","last_peered":"2024-09-13T00:52:39.280246+0000","last_clean":"2024-09-13T00:52:39.280246+0000","last_became_active":"2024-09-13T00:46:45.523318+0000","last_became_peered":"2024-09-13T00:46:45.523318+0000","last_unstale":"2024-09-13T00:52:39.280246+0000","last_undegraded":"2024-09-13T00:52:39.280246+0000","last_fullsized":"2024-09-13T00:52:39.280246+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:39:17.203202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3,"num_read_kb":3,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":892,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.258169+0000","last_change":"2024-09-13T00:32:23.861007+0000","last_active":"2024-09-13T00:52:39.258169+0000","last_peered":"2024-09-13T00:52:39.258169+0000","last_clean":"2024-09-13T00:52:39.258169+0000","last_became_active":"2024-09-13T00:32:23.860213+0000","last_became_peered":"2024-09-13T00:32:23.860213+0000","last_unstale":"2024-09-13T00:52:39.258169+0000","last_undegraded":"2024-09-13T00:52:39.258169+0000","last_fullsized":"2024-09-13T00:52:39.258169+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T11:58:20.866610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"62'1","reported_seq":932,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.280199+0000","last_change":"2024-09-13T00:46:45.502612+0000","last_active":"2024-09-13T00:52:39.280199+0000","last_peered":"2024-09-13T00:52:39.280199+0000","last_clean":"2024-09-13T00:52:39.280199+0000","last_became_active":"2024-09-13T00:46:45.502062+0000","last_became_peered":"2024-09-13T00:46:45.502062+0000","last_unstale":"2024-09-13T00:52:39.280199+0000","last_undegraded":"2024-09-13T00:52:39.280199+0000","last_fullsized":"2024-09-13T00:52:39.280199+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:25:34.771467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"62'8","reported_seq":928,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.430964+0000","last_change":"2024-09-13T00:43:18.190299+0000","last_active":"2024-09-13T00:52:39.430964+0000","last_peered":"2024-09-13T00:52:39.430964+0000","last_clean":"2024-09-13T00:52:39.430964+0000","last_became_active":"2024-09-13T00:43:18.189192+0000","last_became_peered":"2024-09-13T00:43:18.189192+0000","last_unstale":"2024-09-13T00:52:39.430964+0000","last_undegraded":"2024-09-13T00:52:39.430964+0000","last_fullsized":"2024-09-13T00:52:39.430964+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T02:02:26.977888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":12,"num_read_kb":12,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":923,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.320939+0000","last_change":"2024-09-13T00:46:15.349052+0000","last_active":"2024-09-13T00:52:40.320939+0000","last_peered":"2024-09-13T00:52:40.320939+0000","last_clean":"2024-09-13T00:52:40.320939+0000","last_became_active":"2024-09-13T00:46:15.346970+0000","last_became_peered":"2024-09-13T00:46:15.346970+0000","last_unstale":"2024-09-13T00:52:40.320939+0000","last_undegraded":"2024-09-13T00:52:40.320939+0000","last_fullsized":"2024-09-13T00:52:40.320939+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T10:34:13.110080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":913,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.279467+0000","last_change":"2024-09-13T00:46:45.508486+0000","last_active":"2024-09-13T00:52:39.279467+0000","last_peered":"2024-09-13T00:52:39.279467+0000","last_clean":"2024-09-13T00:52:39.279467+0000","last_became_active":"2024-09-13T00:46:45.508135+0000","last_became_peered":"2024-09-13T00:46:45.508135+0000","last_unstale":"2024-09-13T00:52:39.279467+0000","last_undegraded":"2024-09-13T00:52:39.279467+0000","last_fullsized":"2024-09-13T00:52:39.279467+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T04:05:04.774232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":900,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.319010+0000","last_change":"2024-09-13T00:46:15.332428+0000","last_active":"2024-09-13T00:52:40.319010+0000","last_peered":"2024-09-13T00:52:40.319010+0000","last_clean":"2024-09-13T00:52:40.319010+0000","last_became_active":"2024-09-13T00:46:15.331967+0000","last_became_peered":"2024-09-13T00:46:15.331967+0000","last_unstale":"2024-09-13T00:52:40.319010+0000","last_undegraded":"2024-09-13T00:52:40.319010+0000","last_fullsized":"2024-09-13T00:52:40.319010+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T06:02:07.306591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":900,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.276229+0000","last_change":"2024-09-13T00:46:15.356215+0000","last_active":"2024-09-13T00:52:39.276229+0000","last_peered":"2024-09-13T00:52:39.276229+0000","last_clean":"2024-09-13T00:52:39.276229+0000","last_became_active":"2024-09-13T00:46:15.355427+0000","last_became_peered":"2024-09-13T00:46:15.355427+0000","last_unstale":"2024-09-13T00:52:39.276229+0000","last_undegraded":"2024-09-13T00:52:39.276229+0000","last_fullsized":"2024-09-13T00:52:39.276229+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T09:39:06.852517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":909,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.259364+0000","last_change":"2024-09-13T00:46:15.329535+0000","last_active":"2024-09-13T00:52:39.259364+0000","last_peered":"2024-09-13T00:52:39.259364+0000","last_clean":"2024-09-13T00:52:39.259364+0000","last_became_active":"2024-09-13T00:46:15.329152+0000","last_became_peered":"2024-09-13T00:46:15.329152+0000","last_unstale":"2024-09-13T00:52:39.259364+0000","last_undegraded":"2024-09-13T00:52:39.259364+0000","last_fullsized":"2024-09-13T00:52:39.259364+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T05:40:36.773015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"62'2","reported_seq":934,"reported_epoch":955,"state":"active+clean","last_fresh":"2024-09-13T00:52:39.279892+0000","last_change":"2024-09-13T00:46:45.504711+0000","last_active":"2024-09-13T00:52:39.279892+0000","last_peered":"2024-09-13T00:52:39.279892+0000","last_clean":"2024-09-13T00:52:39.279892+0000","last_became_active":"2024-09-13T00:46:45.504345+0000","last_became_peered":"2024-09-13T00:46:45.504345+0000","last_unstale":"2024-09-13T00:52:39.279892+0000","last_undegraded":"2024-09-13T00:52:39.279892+0000","last_fullsized":"2024-09-13T00:52:39.279892+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":59,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:22.061999+0000","last_clean_scrub_stamp":"2024-09-13T00:32:22.061999+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T03:34:15.943161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":7,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":909,"reported_epoch":956,"state":"active+clean","last_fresh":"2024-09-13T00:52:40.320699+0000","last_change":"2024-09-13T00:46:15.350681+0000","last_active":"2024-09-13T00:52:40.320699+0000","last_peered":"2024-09-13T00:52:40.320699+0000","last_clean":"2024-09-13T00:52:40.320699+0000","last_became_active":"2024-09-13T00:46:15.350293+0000","last_became_peered":"2024-09-13T00:46:15.350293+0000","last_unstale":"2024-09-13T00:52:40.320699+0000","last_undegraded":"2024-09-13T00:52:40.320699+0000","last_fullsized":"2024-09-13T00:52:40.320699+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-13T00:32:23.071547+0000","last_clean_scrub_stamp":"2024-09-13T00:32:23.071547+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-14T12:31:39.094477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":414,"num_read_kb":849,"num_write":173,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112,"ondisk_log_size":112,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5536,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":38,"num_read_kb":40,"num_write":62,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5904,"internal_metadata":0},"log_size":62,"ondisk_log_size":62,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":48,"seq":206158430522,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":321744,"kb_used_data":16068,"kb_used_omap":1,"kb_used_meta":305662,"kb_avail":93423408,"statfs":{"total":95995035648,"available":95665569792,"internally_reserved":0,"allocated":16453632,"data_stored":10508274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":312998232},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:53:47 2024","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.83999999999999997,"15min":0.84499999999999997},"min":{"1min":0.52200000000000002,"5min":0.44700000000000001,"15min":0.38},"max":{"1min":1.167,"5min":1.2529999999999999,"15min":2.8250000000000002},"last":0.82199999999999995},{"interface":"front","average":{"1min":0.91600000000000004,"5min":1.1379999999999999,"15min":1.0229999999999999},"min":{"1min":0.53300000000000003,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.21,"5min":16.599,"15min":16.599},"last":1.1599999999999999}]},{"osd":1,"last update":"Fri Sep 13 00:54:09 2024","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.0669999999999999,"15min":1.0840000000000001},"min":{"1min":0.64100000000000001,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":1.286,"5min":1.53,"15min":3.9239999999999999},"last":1.109},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.0760000000000001,"15min":1.0369999999999999},"min":{"1min":0.64200000000000002,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.3220000000000001,"5min":1.5880000000000001,"15min":1.5880000000000001},"last":1.1439999999999999}]},{"osd":2,"last update":"Fri Sep 13 00:53:53 2024","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.99099999999999999,"15min":0.91300000000000003},"min":{"1min":0.71499999999999997,"5min":0.54500000000000004,"15min":0.53900000000000003},"max":{"1min":1.2330000000000001,"5min":2.0760000000000001,"15min":2.0760000000000001},"last":0.876},{"interface":"front","average":{"1min":0.98799999999999999,"5min":1.032,"15min":0.98599999999999999},"min":{"1min":0.72999999999999998,"5min":0.377,"15min":0.377},"max":{"1min":1.323,"5min":1.458,"15min":3.8479999999999999},"last":1.214}]},{"osd":3,"last update":"Fri Sep 13 00:53:25 2024","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.94399999999999995,"15min":0.94599999999999995},"min":{"1min":0.69799999999999995,"5min":0.56399999999999995,"15min":0.48499999999999999},"max":{"1min":1.1000000000000001,"5min":1.97,"15min":2.96},"last":1.0469999999999999},{"interface":"front","average":{"1min":1.016,"5min":1.089,"15min":1.0269999999999999},"min":{"1min":0.72399999999999998,"5min":0.623,"15min":0.58599999999999997},"max":{"1min":1.282,"5min":2.3359999999999999,"15min":2.3359999999999999},"last":1.0740000000000001}]},{"osd":4,"last update":"Fri Sep 13 00:53:53 2024","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.0289999999999999,"15min":0.99299999999999999},"min":{"1min":0.83299999999999996,"5min":0.60099999999999998,"15min":0.54500000000000004},"max":{"1min":1.3100000000000001,"5min":2.4319999999999999,"15min":2.4319999999999999},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.077,"15min":1.1699999999999999},"min":{"1min":0.68600000000000005,"5min":0.58599999999999997,"15min":0.56399999999999995},"max":{"1min":1.365,"5min":3.2530000000000001,"15min":3.6579999999999999},"last":0.76700000000000002}]},{"osd":5,"last update":"Fri Sep 13 00:54:09 2024","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.075,"15min":1.0209999999999999},"min":{"1min":0.78100000000000003,"5min":0.621,"15min":0.38800000000000001},"max":{"1min":1.3149999999999999,"5min":2.863,"15min":2.863},"last":0.97999999999999998},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.1879999999999999,"15min":1.103},"min":{"1min":0.78900000000000003,"5min":0.59799999999999998,"15min":0.46000000000000002},"max":{"1min":1.3049999999999999,"5min":5.4960000000000004,"15min":5.4960000000000004},"last":1.0089999999999999}]},{"osd":6,"last update":"Fri Sep 13 00:54:09 2024","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.2030000000000001,"15min":1.173},"min":{"1min":0.96799999999999997,"5min":0.67100000000000004,"15min":0.36899999999999999},"max":{"1min":1.3520000000000001,"5min":2.2970000000000002,"15min":5.8849999999999998},"last":0.91100000000000003},{"interface":"front","average":{"1min":1.032,"5min":1.0169999999999999,"15min":1.0760000000000001},"min":{"1min":0.72899999999999998,"5min":0.51100000000000001,"15min":0.40699999999999997},"max":{"1min":1.339,"5min":1.413,"15min":5.6349999999999998},"last":0.93700000000000006}]}]},{"osd":6,"up_from":42,"seq":180388626756,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":336892,"kb_used_data":16052,"kb_used_omap":1,"kb_used_meta":309822,"kb_avail":93408260,"statfs":{"total":95995035648,"available":95650058240,"internally_reserved":0,"allocated":16437248,"data_stored":10507588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":317258072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:53:35 2024","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.026,"15min":0.97899999999999998},"min":{"1min":0.54400000000000004,"5min":0.443,"15min":0.33700000000000002},"max":{"1min":1.4450000000000001,"5min":4.3140000000000001,"15min":8.3529999999999998},"last":1.169},{"interface":"front","average":{"1min":1.0329999999999999,"5min":0.97699999999999998,"15min":0.97199999999999998},"min":{"1min":0.45800000000000002,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.55,"5min":4.4160000000000004,"15min":8.423},"last":0.84599999999999997}]},{"osd":1,"last update":"Fri Sep 13 00:54:19 2024","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":0.96099999999999997,"15min":0.95399999999999996},"min":{"1min":0.47599999999999998,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":1.155,"5min":5.0010000000000003,"15min":5.0010000000000003},"last":0.78700000000000003},{"interface":"front","average":{"1min":0.97199999999999998,"5min":1.0920000000000001,"15min":1.0309999999999999},"min":{"1min":0.79400000000000004,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.2030000000000001,"5min":8.1210000000000004,"15min":8.1210000000000004},"last":1.1970000000000001}]},{"osd":2,"last update":"Fri Sep 13 00:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.94799999999999995,"5min":1.1000000000000001,"15min":1.304},"min":{"1min":0.443,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":1.415,"5min":7.9489999999999998,"15min":15.148999999999999},"last":1.266},{"interface":"front","average":{"1min":1.0760000000000001,"5min":1.175,"15min":1.006},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.333,"5min":8.2040000000000006,"15min":8.2040000000000006},"last":1.0169999999999999}]},{"osd":3,"last update":"Fri Sep 13 00:53:29 2024","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":1.022,"15min":1.266},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.48199999999999998},"max":{"1min":1.1870000000000001,"5min":4.5060000000000002,"15min":8.8490000000000002},"last":0.98199999999999998},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.341,"15min":1.274},"min":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.44},"max":{"1min":1.397,"5min":11.970000000000001,"15min":11.970000000000001},"last":0.88500000000000001}]},{"osd":4,"last update":"Fri Sep 13 00:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.81999999999999995,"5min":0.93700000000000006,"15min":0.97299999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.0840000000000001,"5min":5.4219999999999997,"15min":5.4219999999999997},"last":1.0409999999999999},{"interface":"front","average":{"1min":1.131,"5min":1.2030000000000001,"15min":1.145},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":1.3440000000000001,"5min":8.4160000000000004,"15min":8.4160000000000004},"last":0.95499999999999996}]},{"osd":5,"last update":"Fri Sep 13 00:53:35 2024","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":1.2010000000000001,"15min":1.115},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.2190000000000001,"5min":18.739999999999998,"15min":18.739999999999998},"last":0.70999999999999996},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.236,"15min":1.1499999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.32100000000000001},"max":{"1min":1.5189999999999999,"5min":8.3439999999999994,"15min":12.009},"last":1.1279999999999999}]},{"osd":7,"last update":"Fri Sep 13 00:53:21 2024","interfaces":[{"interface":"back","average":{"1min":1.022,"5min":1.1259999999999999,"15min":1.171},"min":{"1min":0.48599999999999999,"5min":0.40999999999999998,"15min":0.379},"max":{"1min":1.7070000000000001,"5min":4.9130000000000003,"15min":20.638000000000002},"last":0.92100000000000004},{"interface":"front","average":{"1min":1.155,"5min":1.242,"15min":1.218},"min":{"1min":0.433,"5min":0.433,"15min":0.42799999999999999},"max":{"1min":1.9399999999999999,"5min":4.766,"15min":9.7750000000000004},"last":1.0960000000000001}]}]},{"osd":5,"up_from":36,"seq":154618822990,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":366880,"kb_used_data":15464,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93378272,"statfs":{"total":95995035648,"available":95619350528,"internally_reserved":0,"allocated":15835136,"data_stored":9913040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:53:41 2024","interfaces":[{"interface":"back","average":{"1min":0.94199999999999995,"5min":1.038,"15min":0.93200000000000005},"min":{"1min":0.66000000000000003,"5min":0.5,"15min":0.46700000000000003},"max":{"1min":1.25,"5min":5.9050000000000002,"15min":5.9050000000000002},"last":1.075},{"interface":"front","average":{"1min":0.88800000000000001,"5min":1.0129999999999999,"15min":1},"min":{"1min":0.629,"5min":0.48999999999999999,"15min":0.48399999999999999},"max":{"1min":1.0880000000000001,"5min":5.79,"15min":8.5899999999999999},"last":0.84899999999999998}]},{"osd":1,"last update":"Fri Sep 13 00:54:14 2024","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":1.0029999999999999,"15min":1.1240000000000001},"min":{"1min":0.67000000000000004,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":1.159,"5min":6.0300000000000002,"15min":7.835},"last":0.96299999999999997},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.054,"15min":1.212},"min":{"1min":0.57299999999999995,"5min":0.443,"15min":0.443},"max":{"1min":1.2150000000000001,"5min":5.5270000000000001,"15min":8.8599999999999994},"last":1.0089999999999999}]},{"osd":2,"last update":"Fri Sep 13 00:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":1.056,"15min":0.98599999999999999},"min":{"1min":0.66800000000000004,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.2330000000000001,"5min":5.8380000000000001,"15min":16.137},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.92400000000000004,"5min":1.131,"15min":1.008},"min":{"1min":0.71499999999999997,"5min":0.52300000000000002,"15min":0.52300000000000002},"max":{"1min":1.129,"5min":11.856,"15min":11.856},"last":0.92600000000000005}]},{"osd":3,"last update":"Fri Sep 13 00:53:23 2024","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.0680000000000001,"15min":1.02},"min":{"1min":0.60199999999999998,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.2629999999999999,"5min":4.0439999999999996,"15min":5.9509999999999996},"last":0.89100000000000001},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.2,"15min":1.1579999999999999},"min":{"1min":0.752,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.2509999999999999,"5min":5.5229999999999997,"15min":10.887},"last":1.1060000000000001}]},{"osd":4,"last update":"Fri Sep 13 00:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.88900000000000001,"5min":1.048,"15min":1.0960000000000001},"min":{"1min":0.64000000000000001,"5min":0.54900000000000004,"15min":0.54900000000000004},"max":{"1min":1.222,"5min":13.478999999999999,"15min":13.478999999999999},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.139,"15min":1.179},"min":{"1min":0.66500000000000004,"5min":0.59699999999999998,"15min":0.59699999999999998},"max":{"1min":1.286,"5min":5.7469999999999999,"15min":5.7469999999999999},"last":1.0429999999999999}]},{"osd":6,"last update":"Fri Sep 13 00:53:23 2024","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.2929999999999999,"15min":1.125},"min":{"1min":0.66500000000000004,"5min":0.629,"15min":0.34100000000000003},"max":{"1min":1.4370000000000001,"5min":12.81,"15min":12.81},"last":1.218},{"interface":"front","average":{"1min":1.139,"5min":1.3799999999999999,"15min":1.218},"min":{"1min":0.873,"5min":0.65500000000000003,"15min":0.36199999999999999},"max":{"1min":1.4390000000000001,"5min":12.98,"15min":12.98},"last":1.137}]},{"osd":7,"last update":"Fri Sep 13 00:54:14 2024","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.387,"15min":1.3109999999999999},"min":{"1min":0.96099999999999997,"5min":0.68400000000000005,"15min":0.39700000000000002},"max":{"1min":1.3400000000000001,"5min":13.388999999999999,"15min":13.388999999999999},"last":1.177},{"interface":"front","average":{"1min":1.079,"5min":1.331,"15min":1.2070000000000001},"min":{"1min":0.82699999999999996,"5min":0.67100000000000004,"15min":0.439},"max":{"1min":1.4099999999999999,"5min":12.470000000000001,"15min":12.470000000000001},"last":1.157}]}]},{"osd":4,"up_from":649,"seq":2787433775197,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":69444,"kb_used_data":15484,"kb_used_omap":67,"kb_used_meta":53884,"kb_avail":93675708,"statfs":{"total":95995035648,"available":95923924992,"internally_reserved":0,"allocated":15855616,"data_stored":9917836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":68780,"internal_metadata":55178068},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.97799999999999998,"15min":1.0489999999999999},"min":{"1min":0.55600000000000005,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.1859999999999999,"5min":3.8359999999999999,"15min":4.7279999999999998},"last":0.626},{"interface":"front","average":{"1min":0.93700000000000006,"5min":1.0049999999999999,"15min":1.0600000000000001},"min":{"1min":0.49199999999999999,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":1.27,"5min":3.6779999999999999,"15min":3.6779999999999999},"last":0.67700000000000005}]},{"osd":1,"last update":"Fri Sep 13 00:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.0089999999999999,"15min":0.996},"min":{"1min":0.73899999999999999,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.3300000000000001,"5min":4.8390000000000004,"15min":6.9649999999999999},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.89600000000000002,"5min":1.0289999999999999,"15min":1.0860000000000001},"min":{"1min":0.63500000000000001,"5min":0.58499999999999996,"15min":0.56799999999999995},"max":{"1min":1.2170000000000001,"5min":3.4159999999999999,"15min":5.7240000000000002},"last":0.57599999999999996}]},{"osd":2,"last update":"Fri Sep 13 00:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.98499999999999999,"15min":1.04},"min":{"1min":0.61099999999999999,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.2589999999999999,"5min":2.8650000000000002,"15min":6.9349999999999996},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.93500000000000005,"5min":1.121,"15min":1.1160000000000001},"min":{"1min":0.55500000000000005,"5min":0.45700000000000002,"15min":0.41599999999999998},"max":{"1min":1.2270000000000001,"5min":6.5810000000000004,"15min":6.5810000000000004},"last":0.77900000000000003}]},{"osd":3,"last update":"Fri Sep 13 00:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":1.0529999999999999,"15min":1.127},"min":{"1min":0.58499999999999996,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.2130000000000001,"5min":3.7639999999999998,"15min":3.7639999999999998},"last":0.85499999999999998},{"interface":"front","average":{"1min":0.92300000000000004,"5min":1.042,"15min":1.0640000000000001},"min":{"1min":0.64300000000000002,"5min":0.379,"15min":0.379},"max":{"1min":1.216,"5min":7.7889999999999997,"15min":7.7889999999999997},"last":0.65300000000000002}]},{"osd":5,"last update":"Fri Sep 13 00:53:58 2024","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.0620000000000001,"15min":1.075},"min":{"1min":0.72099999999999997,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.3440000000000001,"5min":10.381,"15min":10.381},"last":0.88400000000000001},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.034,"15min":1.046},"min":{"1min":0.80900000000000005,"5min":0.374,"15min":0.374},"max":{"1min":1.3799999999999999,"5min":5.0259999999999998,"15min":5.0259999999999998},"last":1.004}]},{"osd":6,"last update":"Fri Sep 13 00:53:58 2024","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.147,"15min":1.0980000000000001},"min":{"1min":0.77000000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.46,"5min":10.474,"15min":10.474},"last":0.80500000000000005},{"interface":"front","average":{"1min":1.157,"5min":1.157,"15min":1.1679999999999999},"min":{"1min":0.85599999999999998,"5min":0.433,"15min":0.433},"max":{"1min":1.4410000000000001,"5min":4.9459999999999997,"15min":6.6159999999999997},"last":0.91700000000000004}]},{"osd":7,"last update":"Fri Sep 13 00:53:58 2024","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.125,"15min":1.21},"min":{"1min":0.68799999999999994,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.502,"5min":3.2749999999999999,"15min":3.3940000000000001},"last":0.90500000000000003},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.2649999999999999,"15min":1.2609999999999999},"min":{"1min":0.878,"5min":0.505,"15min":0.505},"max":{"1min":1.407,"5min":10.548,"15min":12.227},"last":0.95699999999999996}]}]},{"osd":3,"up_from":643,"seq":2761663971427,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":72244,"kb_used_data":15464,"kb_used_omap":74,"kb_used_meta":45685,"kb_avail":93672908,"statfs":{"total":95995035648,"available":95921057792,"internally_reserved":0,"allocated":15835136,"data_stored":9913030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76118,"internal_metadata":46782122},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:53:35 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.82899999999999996,"15min":3.4780000000000002},"min":{"1min":0.33300000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.93400000000000005,"5min":14.706,"15min":92.156000000000006},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.75900000000000001,"5min":0.81200000000000006,"15min":2.9129999999999998},"min":{"1min":0.48199999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.1899999999999999,"5min":7.4950000000000001,"15min":72.637},"last":0.313}]},{"osd":1,"last update":"Fri Sep 13 00:53:35 2024","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.78000000000000003,"15min":3.1080000000000001},"min":{"1min":0.36899999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.0069999999999999,"5min":1.5169999999999999,"15min":75.415999999999997},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.69599999999999995,"5min":0.73399999999999999,"15min":2.4079999999999999},"min":{"1min":0.441,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.3109999999999999,"5min":2.0139999999999998,"15min":59.807000000000002},"last":0.40300000000000002}]},{"osd":2,"last update":"Fri Sep 13 00:53:35 2024","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.83499999999999996,"15min":2.0640000000000001},"min":{"1min":0.45300000000000001,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.044,"5min":3.1930000000000001,"15min":26.347999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.878,"5min":0.96499999999999997,"15min":3.2010000000000001},"min":{"1min":0.51500000000000001,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.153,"5min":3.3860000000000001,"15min":75.534000000000006},"last":0.46600000000000003}]},{"osd":4,"last update":"Fri Sep 13 00:54:01 2024","interfaces":[{"interface":"back","average":{"1min":1.0129999999999999,"5min":1.0429999999999999,"15min":1.466},"min":{"1min":0.65400000000000003,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.2210000000000001,"5min":3.391,"15min":9.6010000000000009},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.028,"15min":1.3009999999999999},"min":{"1min":0.60399999999999998,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.2809999999999999,"5min":2.2210000000000001,"15min":9.3070000000000004},"last":0.84199999999999997}]},{"osd":5,"last update":"Fri Sep 13 00:53:35 2024","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.0089999999999999,"15min":2.7429999999999999},"min":{"1min":0.60399999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.28,"5min":2.9100000000000001,"15min":59.741},"last":0.81100000000000005},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.1319999999999999,"15min":2.9119999999999999},"min":{"1min":0.66200000000000003,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.5109999999999999,"5min":9.8919999999999995,"15min":60.402000000000001},"last":0.73099999999999998}]},{"osd":6,"last update":"Fri Sep 13 00:53:35 2024","interfaces":[{"interface":"back","average":{"1min":1.04,"5min":1.075,"15min":3.2839999999999998},"min":{"1min":0.55300000000000005,"5min":0.44,"15min":0.44},"max":{"1min":1.4399999999999999,"5min":1.8340000000000001,"15min":75.468999999999994},"last":0.79800000000000004},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.1200000000000001,"15min":3.3980000000000001},"min":{"1min":0.746,"5min":0.499,"15min":0.499},"max":{"1min":1.365,"5min":3.46,"15min":75.626999999999995},"last":0.71899999999999997}]},{"osd":7,"last update":"Fri Sep 13 00:53:35 2024","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.0920000000000001,"15min":1.702},"min":{"1min":0.63700000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":1.333,"5min":2.266,"15min":16.564},"last":0.627},{"interface":"front","average":{"1min":1.091,"5min":1.1799999999999999,"15min":4.0720000000000001},"min":{"1min":0.76100000000000001,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":1.4810000000000001,"5min":5.1289999999999996,"15min":92.200999999999993},"last":0.76000000000000001}]}]},{"osd":2,"up_from":629,"seq":2701534429290,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":105976,"kb_used_data":15464,"kb_used_omap":72,"kb_used_meta":58167,"kb_avail":93639176,"statfs":{"total":95995035648,"available":95886516224,"internally_reserved":0,"allocated":15835136,"data_stored":9909549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":74464,"internal_metadata":59563296},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:53:52 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.83299999999999996,"15min":0.66600000000000004},"min":{"1min":0.40100000000000002,"5min":0.224,"15min":0.20000000000000001},"max":{"1min":0.89400000000000002,"5min":13.68,"15min":13.68},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.66800000000000004,"5min":0.97199999999999998,"15min":0.79800000000000004},"min":{"1min":0.36399999999999999,"5min":0.27500000000000002,"15min":0.22800000000000001},"max":{"1min":0.88800000000000001,"5min":15.130000000000001,"15min":15.130000000000001},"last":0.86699999999999999}]},{"osd":1,"last update":"Fri Sep 13 00:53:52 2024","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.79400000000000004,"15min":0.69699999999999995},"min":{"1min":0.42199999999999999,"5min":0.314,"15min":0.249},"max":{"1min":1.032,"5min":4.0940000000000003,"15min":4.0940000000000003},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.877,"15min":0.76600000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.26600000000000001},"max":{"1min":1.095,"5min":6.3029999999999999,"15min":6.3029999999999999},"last":0.73599999999999999}]},{"osd":3,"last update":"Fri Sep 13 00:53:28 2024","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":0.86099999999999999,"15min":0.84699999999999998},"min":{"1min":0.40999999999999998,"5min":0.36599999999999999,"15min":0.29099999999999998},"max":{"1min":1.1950000000000001,"5min":2.573,"15min":11.715999999999999},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.91700000000000004,"5min":0.95599999999999996,"15min":0.85699999999999998},"min":{"1min":0.46500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.0379999999999998,"5min":3.0699999999999998,"15min":3.504},"last":0.77500000000000002}]},{"osd":4,"last update":"Fri Sep 13 00:53:52 2024","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":1.0940000000000001,"15min":1.52},"min":{"1min":0.68000000000000005,"5min":0.55800000000000005,"15min":0.47199999999999998},"max":{"1min":1.244,"5min":4.3949999999999996,"15min":12.356999999999999},"last":0.93100000000000005},{"interface":"front","average":{"1min":0.95299999999999996,"5min":1.226,"15min":1.234},"min":{"1min":0.64900000000000002,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.2889999999999999,"5min":10.207000000000001,"15min":10.207000000000001},"last":0.94599999999999995}]},{"osd":5,"last update":"Fri Sep 13 00:53:52 2024","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":1.1479999999999999,"15min":1.022},"min":{"1min":0.63700000000000001,"5min":0.58799999999999997,"15min":0.47899999999999998},"max":{"1min":1.2070000000000001,"5min":6.4630000000000001,"15min":7.625},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.234,"15min":1.091},"min":{"1min":0.65700000000000003,"5min":0.44600000000000001,"15min":0.40500000000000003},"max":{"1min":1.2849999999999999,"5min":15.185,"15min":15.185},"last":1.0740000000000001}]},{"osd":6,"last update":"Fri Sep 13 00:53:52 2024","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.238,"15min":1.101},"min":{"1min":0.66300000000000003,"5min":0.57799999999999996,"15min":0.44800000000000001},"max":{"1min":1.2430000000000001,"5min":6.1529999999999996,"15min":12.483000000000001},"last":1.024},{"interface":"front","average":{"1min":1.036,"5min":1.27,"15min":1.0880000000000001},"min":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.53400000000000003},"max":{"1min":1.3320000000000001,"5min":7.9480000000000004,"15min":7.9480000000000004},"last":1.3080000000000001}]},{"osd":7,"last update":"Fri Sep 13 00:53:52 2024","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.3129999999999999,"15min":1.103},"min":{"1min":0.64500000000000002,"5min":0.49199999999999999,"15min":0.49199999999999999},"max":{"1min":1.3160000000000001,"5min":11.964,"15min":11.964},"last":1.2490000000000001},{"interface":"front","average":{"1min":1.036,"5min":1.2110000000000001,"15min":1.119},"min":{"1min":0.67800000000000005,"5min":0.56699999999999995,"15min":0.53500000000000003},"max":{"1min":1.2430000000000001,"5min":4.7149999999999999,"15min":7.5350000000000001},"last":1.099}]}]},{"osd":1,"up_from":539,"seq":2314987372671,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":69292,"kb_used_data":15456,"kb_used_omap":53,"kb_used_meta":53770,"kb_avail":93675860,"statfs":{"total":95995035648,"available":95924080640,"internally_reserved":0,"allocated":15826944,"data_stored":9906981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":54760,"internal_metadata":55061016},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Sep 13 00:54:18 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.71899999999999997,"15min":0.72299999999999998},"min":{"1min":0.32000000000000001,"5min":0.29699999999999999,"15min":0.252},"max":{"1min":1.0069999999999999,"5min":2.056,"15min":2.778},"last":0.501},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.77500000000000002,"15min":0.79500000000000004},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29399999999999998},"max":{"1min":1.0860000000000001,"5min":2.4399999999999999,"15min":9.625},"last":0.68500000000000005}]},{"osd":2,"last update":"Fri Sep 13 00:53:51 2024","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":0.71599999999999997,"15min":0.67900000000000005},"min":{"1min":0.39300000000000002,"5min":0.27400000000000002,"15min":0.20399999999999999},"max":{"1min":1.395,"5min":1.4490000000000001,"15min":2.7330000000000001},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.80200000000000005,"15min":0.73799999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":1.0169999999999999,"5min":1.399,"15min":2.2629999999999999},"last":1.0009999999999999}]},{"osd":3,"last update":"Fri Sep 13 00:53:29 2024","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.85999999999999999,"15min":0.92300000000000004},"min":{"1min":0.441,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.345,"5min":1.345,"15min":5.2300000000000004},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.90500000000000003,"15min":0.89200000000000002},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.4319999999999999,"5min":1.4870000000000001,"15min":1.889},"last":0.92300000000000004}]},{"osd":4,"last update":"Fri Sep 13 00:54:05 2024","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":1.018,"15min":1.6910000000000001},"min":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.54400000000000004},"max":{"1min":1.2310000000000001,"5min":1.641,"15min":16.684999999999999},"last":1.0960000000000001},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.1100000000000001,"15min":1.8280000000000001},"min":{"1min":0.58499999999999996,"5min":0.48699999999999999,"15min":0.48099999999999998},"max":{"1min":4.1820000000000004,"5min":5.9100000000000001,"15min":24.937999999999999},"last":0.78400000000000003}]},{"osd":5,"last update":"Fri Sep 13 00:54:18 2024","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":1.0249999999999999,"15min":1.155},"min":{"1min":0.59699999999999998,"5min":0.53700000000000003,"15min":0.42299999999999999},"max":{"1min":1.2709999999999999,"5min":3.673,"15min":29.626000000000001},"last":0.89300000000000002},{"interface":"front","average":{"1min":1.0389999999999999,"5min":1.1379999999999999,"15min":1.119},"min":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.498},"max":{"1min":1.8500000000000001,"5min":3.9390000000000001,"15min":5.7930000000000001},"last":0.95499999999999996}]},{"osd":6,"last update":"Fri Sep 13 00:54:18 2024","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.075,"15min":1.1180000000000001},"min":{"1min":0.57999999999999996,"5min":0.501,"15min":0.501},"max":{"1min":4.133,"5min":4.133,"15min":9.2550000000000008},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.073,"5min":1.117,"15min":1.1950000000000001},"min":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.52100000000000002},"max":{"1min":1.5069999999999999,"5min":1.8260000000000001,"15min":12.065},"last":1.147}]},{"osd":7,"last update":"Fri Sep 13 00:54:18 2024","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.339,"15min":1.2649999999999999},"min":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.497},"max":{"1min":1.5529999999999999,"5min":12.346,"15min":12.346},"last":1.0369999999999999},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.2589999999999999,"15min":1.2509999999999999},"min":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.53200000000000003},"max":{"1min":1.3180000000000001,"5min":7.1050000000000004,"15min":7.1050000000000004},"last":1.0609999999999999}]}]},{"osd":0,"up_from":502,"seq":2156073582727,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":111724,"kb_used_data":16048,"kb_used_omap":55,"kb_used_meta":53832,"kb_avail":93633428,"statfs":{"total":95995035648,"available":95880630272,"internally_reserved":0,"allocated":16433152,"data_stored":10503900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":56531,"internal_metadata":55124781},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Sep 13 00:54:21 2024","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":1.002,"15min":0.877},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.30199999999999999},"max":{"1min":0.98499999999999999,"5min":24.273,"15min":24.273},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.85399999999999998,"5min":0.84699999999999998,"15min":0.878},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.29199999999999998},"max":{"1min":1.1779999999999999,"5min":1.2350000000000001,"15min":11.792999999999999},"last":1.105}]},{"osd":2,"last update":"Fri Sep 13 00:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.78100000000000003,"15min":0.92800000000000005},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28299999999999997},"max":{"1min":1.075,"5min":1.7230000000000001,"15min":8.4969999999999999},"last":0.82399999999999995},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.77200000000000002,"15min":1.5880000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":1.4399999999999999,"5min":2.2040000000000002,"15min":32.914000000000001},"last":0.73799999999999999}]},{"osd":3,"last update":"Fri Sep 13 00:53:27 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.89300000000000002,"15min":1.0489999999999999},"min":{"1min":0.65800000000000003,"5min":0.48199999999999998,"15min":0.33900000000000002},"max":{"1min":1.647,"5min":1.7689999999999999,"15min":8.7520000000000007},"last":0.94199999999999995},{"interface":"front","average":{"1min":0.81200000000000006,"5min":1.1559999999999999,"15min":1.1850000000000001},"min":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.309},"max":{"1min":1.226,"5min":24.571999999999999,"15min":24.571999999999999},"last":0.88900000000000001}]},{"osd":4,"last update":"Fri Sep 13 00:54:02 2024","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.153,"15min":1.0569999999999999},"min":{"1min":0.73499999999999999,"5min":0.64000000000000001,"15min":0.64000000000000001},"max":{"1min":1.655,"5min":13.815,"15min":13.815},"last":0.80000000000000004},{"interface":"front","average":{"1min":1.0529999999999999,"5min":1.1859999999999999,"15min":1.081},"min":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"max":{"1min":1.468,"5min":13.696999999999999,"15min":13.696999999999999},"last":1.056}]},{"osd":5,"last update":"Fri Sep 13 00:53:43 2024","interfaces":[{"interface":"back","average":{"1min":1.014,"5min":1.323,"15min":1.1799999999999999},"min":{"1min":0.79000000000000004,"5min":0.67600000000000005,"15min":0.47899999999999998},"max":{"1min":1.3979999999999999,"5min":24.376999999999999,"15min":24.376999999999999},"last":1.0209999999999999},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.24,"15min":1.1419999999999999},"min":{"1min":0.80800000000000005,"5min":0.752,"15min":0.57699999999999996},"max":{"1min":1.536,"5min":14.015000000000001,"15min":19.343},"last":1.141}]},{"osd":6,"last update":"Fri Sep 13 00:53:43 2024","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.4179999999999999,"15min":1.2030000000000001},"min":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.57699999999999996},"max":{"1min":1.589,"5min":14.15,"15min":14.15},"last":1},{"interface":"front","average":{"1min":1.119,"5min":1.141,"15min":1.095},"min":{"1min":0.92900000000000005,"5min":0.86399999999999999,"15min":0.56000000000000005},"max":{"1min":1.659,"5min":2.3719999999999999,"15min":8.7870000000000008},"last":1.5209999999999999}]},{"osd":7,"last update":"Fri Sep 13 00:53:43 2024","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.171,"15min":1.133},"min":{"1min":0.87,"5min":0.80500000000000005,"15min":0.58899999999999997},"max":{"1min":1.758,"5min":3.6139999999999999,"15min":9.7620000000000005},"last":1.5760000000000001},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.452,"15min":1.3180000000000001},"min":{"1min":0.93300000000000005,"5min":0.66900000000000004,"15min":0.60499999999999998},"max":{"1min":1.5009999999999999,"5min":24.452000000000002,"15min":28.82},"last":0.97599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2404,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3500,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4820,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-13T00:54:25.167 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-13T00:54:25.167 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-13T00:54:25.167 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-13T00:54:25.167 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:54:25.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:25 smithi077 bash[80681]: audit 2024-09-13T00:54:24.263933+0000 mgr.y (mgr.25304) 557 : audit [DBG] from='client.34809 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:54:25.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:25 smithi077 bash[83127]: audit 2024-09-13T00:54:24.263933+0000 mgr.y (mgr.25304) 557 : audit [DBG] from='client.34809 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:54:25.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:25 smithi088 bash[68960]: audit 2024-09-13T00:54:24.263933+0000 mgr.y (mgr.25304) 557 : audit [DBG] from='client.34809 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-13T00:54:26.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:26 smithi077 bash[80681]: cluster 2024-09-13T00:54:24.908095+0000 mgr.y (mgr.25304) 558 : cluster [DBG] pgmap v1132: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:26 smithi077 bash[83127]: cluster 2024-09-13T00:54:24.908095+0000 mgr.y (mgr.25304) 558 : cluster [DBG] pgmap v1132: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:26 smithi088 bash[68960]: cluster 2024-09-13T00:54:24.908095+0000 mgr.y (mgr.25304) 558 : cluster [DBG] pgmap v1132: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:28.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:28 smithi077 bash[80681]: cluster 2024-09-13T00:54:26.908971+0000 mgr.y (mgr.25304) 559 : cluster [DBG] pgmap v1133: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:28.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:28 smithi077 bash[83127]: cluster 2024-09-13T00:54:26.908971+0000 mgr.y (mgr.25304) 559 : cluster [DBG] pgmap v1133: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:28.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:28 smithi088 bash[68960]: cluster 2024-09-13T00:54:26.908971+0000 mgr.y (mgr.25304) 559 : cluster [DBG] pgmap v1133: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:28.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:54:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:54:28] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-09-13T00:54:29.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:29 smithi077 bash[80681]: audit 2024-09-13T00:54:28.431657+0000 mon.a (mon.0) 1532 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:54:29.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:29 smithi077 bash[83127]: audit 2024-09-13T00:54:28.431657+0000 mon.a (mon.0) 1532 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:54:29.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:29 smithi088 bash[68960]: audit 2024-09-13T00:54:28.431657+0000 mon.a (mon.0) 1532 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:54:29.953 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:54:30.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:30 smithi077 bash[80681]: cluster 2024-09-13T00:54:28.909793+0000 mgr.y (mgr.25304) 560 : cluster [DBG] pgmap v1134: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:30.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:30 smithi077 bash[83127]: cluster 2024-09-13T00:54:28.909793+0000 mgr.y (mgr.25304) 560 : cluster [DBG] pgmap v1134: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:30.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:30 smithi088 bash[68960]: cluster 2024-09-13T00:54:28.909793+0000 mgr.y (mgr.25304) 560 : cluster [DBG] pgmap v1134: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:32.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:32 smithi077 bash[80681]: cluster 2024-09-13T00:54:30.910836+0000 mgr.y (mgr.25304) 561 : cluster [DBG] pgmap v1135: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:32.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:32 smithi077 bash[83127]: cluster 2024-09-13T00:54:30.910836+0000 mgr.y (mgr.25304) 561 : cluster [DBG] pgmap v1135: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:32.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:32 smithi088 bash[68960]: cluster 2024-09-13T00:54:30.910836+0000 mgr.y (mgr.25304) 561 : cluster [DBG] pgmap v1135: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:32.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:54:32 smithi088 bash[61791]: ts=2024-09-13T00:54:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:54:34.004 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:54:34.004 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:54:34.278 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:34 smithi077 bash[80681]: cluster 2024-09-13T00:54:32.911758+0000 mgr.y (mgr.25304) 562 : cluster [DBG] pgmap v1136: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:34.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:34 smithi077 bash[80681]: audit 2024-09-13T00:54:34.006956+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.77:0/2780017058' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:54:34.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:34 smithi077 bash[83127]: cluster 2024-09-13T00:54:32.911758+0000 mgr.y (mgr.25304) 562 : cluster [DBG] pgmap v1136: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:34.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:34 smithi077 bash[83127]: audit 2024-09-13T00:54:34.006956+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.77:0/2780017058' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:54:34.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:34 smithi088 bash[68960]: cluster 2024-09-13T00:54:32.911758+0000 mgr.y (mgr.25304) 562 : cluster [DBG] pgmap v1136: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:34.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:34 smithi088 bash[68960]: audit 2024-09-13T00:54:34.006956+0000 mon.c (mon.2) 56 : audit [DBG] from='client.? 172.21.15.77:0/2780017058' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:54:36.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:36 smithi077 bash[83127]: cluster 2024-09-13T00:54:34.912860+0000 mgr.y (mgr.25304) 563 : cluster [DBG] pgmap v1137: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:36.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:36 smithi077 bash[80681]: cluster 2024-09-13T00:54:34.912860+0000 mgr.y (mgr.25304) 563 : cluster [DBG] pgmap v1137: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:36.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:36 smithi088 bash[68960]: cluster 2024-09-13T00:54:34.912860+0000 mgr.y (mgr.25304) 563 : cluster [DBG] pgmap v1137: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:37.744 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:54:38.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:38 smithi077 bash[80681]: cluster 2024-09-13T00:54:36.913725+0000 mgr.y (mgr.25304) 564 : cluster [DBG] pgmap v1138: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:38 smithi077 bash[83127]: cluster 2024-09-13T00:54:36.913725+0000 mgr.y (mgr.25304) 564 : cluster [DBG] pgmap v1138: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:38.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:38 smithi088 bash[68960]: cluster 2024-09-13T00:54:36.913725+0000 mgr.y (mgr.25304) 564 : cluster [DBG] pgmap v1138: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:38.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:54:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:54:38] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-09-13T00:54:40.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:40 smithi077 bash[80681]: cluster 2024-09-13T00:54:38.914620+0000 mgr.y (mgr.25304) 565 : cluster [DBG] pgmap v1139: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:40.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:40 smithi077 bash[83127]: cluster 2024-09-13T00:54:38.914620+0000 mgr.y (mgr.25304) 565 : cluster [DBG] pgmap v1139: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:40.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:40 smithi088 bash[68960]: cluster 2024-09-13T00:54:38.914620+0000 mgr.y (mgr.25304) 565 : cluster [DBG] pgmap v1139: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:42.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:42 smithi077 bash[80681]: cluster 2024-09-13T00:54:40.915626+0000 mgr.y (mgr.25304) 566 : cluster [DBG] pgmap v1140: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:42.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:42 smithi077 bash[83127]: cluster 2024-09-13T00:54:40.915626+0000 mgr.y (mgr.25304) 566 : cluster [DBG] pgmap v1140: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:42.531 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:54:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:42 smithi088 bash[68960]: cluster 2024-09-13T00:54:40.915626+0000 mgr.y (mgr.25304) 566 : cluster [DBG] pgmap v1140: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:42.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:54:42 smithi088 bash[61791]: ts=2024-09-13T00:54:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:54:44.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:44 smithi077 bash[80681]: cluster 2024-09-13T00:54:42.916500+0000 mgr.y (mgr.25304) 567 : cluster [DBG] pgmap v1141: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:44.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:44 smithi077 bash[80681]: audit 2024-09-13T00:54:43.432083+0000 mon.a (mon.0) 1533 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:54:44.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:44 smithi077 bash[83127]: cluster 2024-09-13T00:54:42.916500+0000 mgr.y (mgr.25304) 567 : cluster [DBG] pgmap v1141: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:44.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:44 smithi077 bash[83127]: audit 2024-09-13T00:54:43.432083+0000 mon.a (mon.0) 1533 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:54:44.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:44 smithi088 bash[68960]: cluster 2024-09-13T00:54:42.916500+0000 mgr.y (mgr.25304) 567 : cluster [DBG] pgmap v1141: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:44.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:44 smithi088 bash[68960]: audit 2024-09-13T00:54:43.432083+0000 mon.a (mon.0) 1533 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:54:46.404 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:54:46.405 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:54:46.416 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:46 smithi077 bash[80681]: cluster 2024-09-13T00:54:44.917478+0000 mgr.y (mgr.25304) 568 : cluster [DBG] pgmap v1142: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:46.416 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:46 smithi077 bash[83127]: cluster 2024-09-13T00:54:44.917478+0000 mgr.y (mgr.25304) 568 : cluster [DBG] pgmap v1142: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:46.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:46 smithi088 bash[68960]: cluster 2024-09-13T00:54:44.917478+0000 mgr.y (mgr.25304) 568 : cluster [DBG] pgmap v1142: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:47.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:47 smithi077 bash[80681]: audit 2024-09-13T00:54:46.407025+0000 mon.a (mon.0) 1534 : audit [DBG] from='client.? 172.21.15.77:0/3648693548' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:54:47.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:47 smithi077 bash[83127]: audit 2024-09-13T00:54:46.407025+0000 mon.a (mon.0) 1534 : audit [DBG] from='client.? 172.21.15.77:0/3648693548' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:54:47.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:47 smithi088 bash[68960]: audit 2024-09-13T00:54:46.407025+0000 mon.a (mon.0) 1534 : audit [DBG] from='client.? 172.21.15.77:0/3648693548' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:54:48.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:48 smithi088 bash[68960]: cluster 2024-09-13T00:54:46.918266+0000 mgr.y (mgr.25304) 569 : cluster [DBG] pgmap v1143: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:48.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:48 smithi077 bash[80681]: cluster 2024-09-13T00:54:46.918266+0000 mgr.y (mgr.25304) 569 : cluster [DBG] pgmap v1143: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:48.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:54:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:54:48] "GET /metrics HTTP/1.1" 200 34261 "" "Prometheus/2.43.0" 2024-09-13T00:54:48.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:48 smithi077 bash[83127]: cluster 2024-09-13T00:54:46.918266+0000 mgr.y (mgr.25304) 569 : cluster [DBG] pgmap v1143: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:50.172 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:54:50.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:50 smithi077 bash[80681]: cluster 2024-09-13T00:54:48.919092+0000 mgr.y (mgr.25304) 570 : cluster [DBG] pgmap v1144: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:50.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:50 smithi077 bash[83127]: cluster 2024-09-13T00:54:48.919092+0000 mgr.y (mgr.25304) 570 : cluster [DBG] pgmap v1144: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:50.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:50 smithi088 bash[68960]: cluster 2024-09-13T00:54:48.919092+0000 mgr.y (mgr.25304) 570 : cluster [DBG] pgmap v1144: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:52.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:52 smithi088 bash[68960]: cluster 2024-09-13T00:54:50.919992+0000 mgr.y (mgr.25304) 571 : cluster [DBG] pgmap v1145: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:52.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:54:52 smithi088 bash[61791]: ts=2024-09-13T00:54:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:54:52.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:52 smithi077 bash[80681]: cluster 2024-09-13T00:54:50.919992+0000 mgr.y (mgr.25304) 571 : cluster [DBG] pgmap v1145: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:52.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:52 smithi077 bash[83127]: cluster 2024-09-13T00:54:50.919992+0000 mgr.y (mgr.25304) 571 : cluster [DBG] pgmap v1145: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:54.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:54 smithi088 bash[68960]: cluster 2024-09-13T00:54:52.920609+0000 mgr.y (mgr.25304) 572 : cluster [DBG] pgmap v1146: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:54.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:54 smithi077 bash[80681]: cluster 2024-09-13T00:54:52.920609+0000 mgr.y (mgr.25304) 572 : cluster [DBG] pgmap v1146: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:54.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:54 smithi077 bash[83127]: cluster 2024-09-13T00:54:52.920609+0000 mgr.y (mgr.25304) 572 : cluster [DBG] pgmap v1146: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:54.958 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:54:56.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:56 smithi077 bash[80681]: cluster 2024-09-13T00:54:54.921525+0000 mgr.y (mgr.25304) 573 : cluster [DBG] pgmap v1147: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:56 smithi077 bash[83127]: cluster 2024-09-13T00:54:54.921525+0000 mgr.y (mgr.25304) 573 : cluster [DBG] pgmap v1147: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:56.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:56 smithi088 bash[68960]: cluster 2024-09-13T00:54:54.921525+0000 mgr.y (mgr.25304) 573 : cluster [DBG] pgmap v1147: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:58.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:58 smithi077 bash[83127]: cluster 2024-09-13T00:54:56.922301+0000 mgr.y (mgr.25304) 574 : cluster [DBG] pgmap v1148: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:58.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:58 smithi077 bash[80681]: cluster 2024-09-13T00:54:56.922301+0000 mgr.y (mgr.25304) 574 : cluster [DBG] pgmap v1148: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:58.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:58 smithi088 bash[68960]: cluster 2024-09-13T00:54:56.922301+0000 mgr.y (mgr.25304) 574 : cluster [DBG] pgmap v1148: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:54:58.626 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:54:58.626 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:54:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:54:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:54:58] "GET /metrics HTTP/1.1" 200 34265 "" "Prometheus/2.43.0" 2024-09-13T00:54:59.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:59 smithi088 bash[68960]: audit 2024-09-13T00:54:58.431531+0000 mon.a (mon.0) 1535 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:54:59.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:54:59 smithi088 bash[68960]: audit 2024-09-13T00:54:58.628392+0000 mon.a (mon.0) 1536 : audit [DBG] from='client.? 172.21.15.77:0/1917165141' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:54:59.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:59 smithi077 bash[83127]: audit 2024-09-13T00:54:58.431531+0000 mon.a (mon.0) 1535 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:54:59.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:54:59 smithi077 bash[83127]: audit 2024-09-13T00:54:58.628392+0000 mon.a (mon.0) 1536 : audit [DBG] from='client.? 172.21.15.77:0/1917165141' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:54:59.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:59 smithi077 bash[80681]: audit 2024-09-13T00:54:58.431531+0000 mon.a (mon.0) 1535 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:54:59.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:54:59 smithi077 bash[80681]: audit 2024-09-13T00:54:58.628392+0000 mon.a (mon.0) 1536 : audit [DBG] from='client.? 172.21.15.77:0/1917165141' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:55:00.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:00 smithi088 bash[68960]: cluster 2024-09-13T00:54:58.923112+0000 mgr.y (mgr.25304) 575 : cluster [DBG] pgmap v1149: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:00.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:00 smithi077 bash[80681]: cluster 2024-09-13T00:54:58.923112+0000 mgr.y (mgr.25304) 575 : cluster [DBG] pgmap v1149: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:00 smithi077 bash[83127]: cluster 2024-09-13T00:54:58.923112+0000 mgr.y (mgr.25304) 575 : cluster [DBG] pgmap v1149: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:02.505 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:55:02.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:02 smithi088 bash[68960]: cluster 2024-09-13T00:55:00.924050+0000 mgr.y (mgr.25304) 576 : cluster [DBG] pgmap v1150: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:02.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:55:02 smithi088 bash[61791]: ts=2024-09-13T00:55:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:55:02.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:02 smithi077 bash[83127]: cluster 2024-09-13T00:55:00.924050+0000 mgr.y (mgr.25304) 576 : cluster [DBG] pgmap v1150: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:02.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:02 smithi077 bash[80681]: cluster 2024-09-13T00:55:00.924050+0000 mgr.y (mgr.25304) 576 : cluster [DBG] pgmap v1150: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:04.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:04 smithi088 bash[68960]: cluster 2024-09-13T00:55:02.924872+0000 mgr.y (mgr.25304) 577 : cluster [DBG] pgmap v1151: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:04.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:04 smithi077 bash[80681]: cluster 2024-09-13T00:55:02.924872+0000 mgr.y (mgr.25304) 577 : cluster [DBG] pgmap v1151: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:04.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:04 smithi077 bash[83127]: cluster 2024-09-13T00:55:02.924872+0000 mgr.y (mgr.25304) 577 : cluster [DBG] pgmap v1151: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:06.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:06 smithi088 bash[68960]: cluster 2024-09-13T00:55:04.925815+0000 mgr.y (mgr.25304) 578 : cluster [DBG] pgmap v1152: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:06.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:06 smithi077 bash[83127]: cluster 2024-09-13T00:55:04.925815+0000 mgr.y (mgr.25304) 578 : cluster [DBG] pgmap v1152: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:06.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:06 smithi077 bash[80681]: cluster 2024-09-13T00:55:04.925815+0000 mgr.y (mgr.25304) 578 : cluster [DBG] pgmap v1152: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:07.289 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:55:08.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:08 smithi088 bash[68960]: cluster 2024-09-13T00:55:06.926588+0000 mgr.y (mgr.25304) 579 : cluster [DBG] pgmap v1153: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:08.635 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:55:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:55:08] "GET /metrics HTTP/1.1" 200 34265 "" "Prometheus/2.43.0" 2024-09-13T00:55:08.635 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:08 smithi077 bash[83127]: cluster 2024-09-13T00:55:06.926588+0000 mgr.y (mgr.25304) 579 : cluster [DBG] pgmap v1153: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:08.635 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:08 smithi077 bash[80681]: cluster 2024-09-13T00:55:06.926588+0000 mgr.y (mgr.25304) 579 : cluster [DBG] pgmap v1153: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:10.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:10 smithi088 bash[68960]: cluster 2024-09-13T00:55:08.927439+0000 mgr.y (mgr.25304) 580 : cluster [DBG] pgmap v1154: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:10.729 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:10 smithi077 bash[80681]: cluster 2024-09-13T00:55:08.927439+0000 mgr.y (mgr.25304) 580 : cluster [DBG] pgmap v1154: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:10.730 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:10 smithi077 bash[83127]: cluster 2024-09-13T00:55:08.927439+0000 mgr.y (mgr.25304) 580 : cluster [DBG] pgmap v1154: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:11.089 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:55:11.089 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:55:11.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:11 smithi077 bash[80681]: audit 2024-09-13T00:55:11.091811+0000 mon.a (mon.0) 1537 : audit [DBG] from='client.? 172.21.15.77:0/658290798' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:55:11.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:11 smithi077 bash[83127]: audit 2024-09-13T00:55:11.091811+0000 mon.a (mon.0) 1537 : audit [DBG] from='client.? 172.21.15.77:0/658290798' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:55:11.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:11 smithi088 bash[68960]: audit 2024-09-13T00:55:11.091811+0000 mon.a (mon.0) 1537 : audit [DBG] from='client.? 172.21.15.77:0/658290798' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:55:12.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:12 smithi088 bash[68960]: cluster 2024-09-13T00:55:10.928057+0000 mgr.y (mgr.25304) 581 : cluster [DBG] pgmap v1155: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:12.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:55:12 smithi088 bash[61791]: ts=2024-09-13T00:55:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:55:12.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:12 smithi077 bash[80681]: cluster 2024-09-13T00:55:10.928057+0000 mgr.y (mgr.25304) 581 : cluster [DBG] pgmap v1155: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:12.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:12 smithi077 bash[83127]: cluster 2024-09-13T00:55:10.928057+0000 mgr.y (mgr.25304) 581 : cluster [DBG] pgmap v1155: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:14 smithi088 bash[68960]: cluster 2024-09-13T00:55:12.928806+0000 mgr.y (mgr.25304) 582 : cluster [DBG] pgmap v1156: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:14.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:14 smithi088 bash[68960]: audit 2024-09-13T00:55:13.432645+0000 mon.a (mon.0) 1538 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:55:14.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:14 smithi077 bash[80681]: cluster 2024-09-13T00:55:12.928806+0000 mgr.y (mgr.25304) 582 : cluster [DBG] pgmap v1156: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:14.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:14 smithi077 bash[80681]: audit 2024-09-13T00:55:13.432645+0000 mon.a (mon.0) 1538 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:55:14.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:14 smithi077 bash[83127]: cluster 2024-09-13T00:55:12.928806+0000 mgr.y (mgr.25304) 582 : cluster [DBG] pgmap v1156: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:14.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:14 smithi077 bash[83127]: audit 2024-09-13T00:55:13.432645+0000 mon.a (mon.0) 1538 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:55:14.856 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:55:16.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:16 smithi088 bash[68960]: cluster 2024-09-13T00:55:14.929845+0000 mgr.y (mgr.25304) 583 : cluster [DBG] pgmap v1157: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:16.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:16 smithi077 bash[80681]: cluster 2024-09-13T00:55:14.929845+0000 mgr.y (mgr.25304) 583 : cluster [DBG] pgmap v1157: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:16.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:16 smithi077 bash[83127]: cluster 2024-09-13T00:55:14.929845+0000 mgr.y (mgr.25304) 583 : cluster [DBG] pgmap v1157: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:18.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:18 smithi088 bash[68960]: cluster 2024-09-13T00:55:16.930741+0000 mgr.y (mgr.25304) 584 : cluster [DBG] pgmap v1158: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:18.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:18 smithi077 bash[80681]: cluster 2024-09-13T00:55:16.930741+0000 mgr.y (mgr.25304) 584 : cluster [DBG] pgmap v1158: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:18.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:55:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:55:18] "GET /metrics HTTP/1.1" 200 34264 "" "Prometheus/2.43.0" 2024-09-13T00:55:18.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:18 smithi077 bash[83127]: cluster 2024-09-13T00:55:16.930741+0000 mgr.y (mgr.25304) 584 : cluster [DBG] pgmap v1158: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:19.646 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:55:20.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:20 smithi077 bash[80681]: cluster 2024-09-13T00:55:18.931571+0000 mgr.y (mgr.25304) 585 : cluster [DBG] pgmap v1159: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:20.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:20 smithi077 bash[83127]: cluster 2024-09-13T00:55:18.931571+0000 mgr.y (mgr.25304) 585 : cluster [DBG] pgmap v1159: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:20.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:20 smithi088 bash[68960]: cluster 2024-09-13T00:55:18.931571+0000 mgr.y (mgr.25304) 585 : cluster [DBG] pgmap v1159: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:22.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:22 smithi088 bash[68960]: cluster 2024-09-13T00:55:20.932467+0000 mgr.y (mgr.25304) 586 : cluster [DBG] pgmap v1160: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:22.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:55:22 smithi088 bash[61791]: ts=2024-09-13T00:55:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:55:22.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:22 smithi077 bash[80681]: cluster 2024-09-13T00:55:20.932467+0000 mgr.y (mgr.25304) 586 : cluster [DBG] pgmap v1160: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:22.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:22 smithi077 bash[83127]: cluster 2024-09-13T00:55:20.932467+0000 mgr.y (mgr.25304) 586 : cluster [DBG] pgmap v1160: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:23.536 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:55:23.536 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:55:24.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:24 smithi077 bash[80681]: cluster 2024-09-13T00:55:22.933264+0000 mgr.y (mgr.25304) 587 : cluster [DBG] pgmap v1161: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:24.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:24 smithi077 bash[80681]: audit 2024-09-13T00:55:23.538175+0000 mon.a (mon.0) 1539 : audit [DBG] from='client.? 172.21.15.77:0/578461236' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:55:24.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:24 smithi077 bash[80681]: audit 2024-09-13T00:55:23.715627+0000 mon.a (mon.0) 1540 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:55:24.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:24 smithi077 bash[83127]: cluster 2024-09-13T00:55:22.933264+0000 mgr.y (mgr.25304) 587 : cluster [DBG] pgmap v1161: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:24.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:24 smithi077 bash[83127]: audit 2024-09-13T00:55:23.538175+0000 mon.a (mon.0) 1539 : audit [DBG] from='client.? 172.21.15.77:0/578461236' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:55:24.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:24 smithi077 bash[83127]: audit 2024-09-13T00:55:23.715627+0000 mon.a (mon.0) 1540 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:55:24.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:24 smithi088 bash[68960]: cluster 2024-09-13T00:55:22.933264+0000 mgr.y (mgr.25304) 587 : cluster [DBG] pgmap v1161: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:24.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:24 smithi088 bash[68960]: audit 2024-09-13T00:55:23.538175+0000 mon.a (mon.0) 1539 : audit [DBG] from='client.? 172.21.15.77:0/578461236' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:55:24.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:24 smithi088 bash[68960]: audit 2024-09-13T00:55:23.715627+0000 mon.a (mon.0) 1540 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:55:26.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:26 smithi077 bash[80681]: cluster 2024-09-13T00:55:24.934217+0000 mgr.y (mgr.25304) 588 : cluster [DBG] pgmap v1162: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:26.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:26 smithi077 bash[83127]: cluster 2024-09-13T00:55:24.934217+0000 mgr.y (mgr.25304) 588 : cluster [DBG] pgmap v1162: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:26.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:26 smithi088 bash[68960]: cluster 2024-09-13T00:55:24.934217+0000 mgr.y (mgr.25304) 588 : cluster [DBG] pgmap v1162: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:27.321 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:55:28.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:28 smithi077 bash[80681]: cluster 2024-09-13T00:55:26.934972+0000 mgr.y (mgr.25304) 589 : cluster [DBG] pgmap v1163: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:28.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:55:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:55:28] "GET /metrics HTTP/1.1" 200 34265 "" "Prometheus/2.43.0" 2024-09-13T00:55:28.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:28 smithi077 bash[83127]: cluster 2024-09-13T00:55:26.934972+0000 mgr.y (mgr.25304) 589 : cluster [DBG] pgmap v1163: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:28.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:28 smithi088 bash[68960]: cluster 2024-09-13T00:55:26.934972+0000 mgr.y (mgr.25304) 589 : cluster [DBG] pgmap v1163: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:29.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:29 smithi077 bash[80681]: audit 2024-09-13T00:55:28.433175+0000 mon.a (mon.0) 1541 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:55:29.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:29 smithi077 bash[80681]: audit 2024-09-13T00:55:29.331723+0000 mon.a (mon.0) 1542 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:55:29.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:29 smithi077 bash[80681]: audit 2024-09-13T00:55:29.338143+0000 mon.a (mon.0) 1543 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:55:29.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:29 smithi077 bash[83127]: audit 2024-09-13T00:55:28.433175+0000 mon.a (mon.0) 1541 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:55:29.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:29 smithi077 bash[83127]: audit 2024-09-13T00:55:29.331723+0000 mon.a (mon.0) 1542 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:55:29.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:29 smithi077 bash[83127]: audit 2024-09-13T00:55:29.338143+0000 mon.a (mon.0) 1543 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:55:29.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:29 smithi088 bash[68960]: audit 2024-09-13T00:55:28.433175+0000 mon.a (mon.0) 1541 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:55:29.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:29 smithi088 bash[68960]: audit 2024-09-13T00:55:29.331723+0000 mon.a (mon.0) 1542 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:55:29.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:29 smithi088 bash[68960]: audit 2024-09-13T00:55:29.338143+0000 mon.a (mon.0) 1543 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:55:30.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:30 smithi077 bash[80681]: cluster 2024-09-13T00:55:28.935443+0000 mgr.y (mgr.25304) 590 : cluster [DBG] pgmap v1164: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:30.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:30 smithi077 bash[80681]: audit 2024-09-13T00:55:29.492501+0000 mon.a (mon.0) 1544 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:55:30.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:30 smithi077 bash[80681]: audit 2024-09-13T00:55:29.504530+0000 mon.a (mon.0) 1545 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:55:30.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:30 smithi077 bash[80681]: audit 2024-09-13T00:55:29.917648+0000 mon.a (mon.0) 1546 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:55:30.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:30 smithi077 bash[80681]: audit 2024-09-13T00:55:29.918919+0000 mon.a (mon.0) 1547 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:55:30.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:30 smithi077 bash[80681]: audit 2024-09-13T00:55:29.930229+0000 mon.a (mon.0) 1548 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:55:30.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:30 smithi077 bash[83127]: cluster 2024-09-13T00:55:28.935443+0000 mgr.y (mgr.25304) 590 : cluster [DBG] pgmap v1164: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:30.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:30 smithi077 bash[83127]: audit 2024-09-13T00:55:29.492501+0000 mon.a (mon.0) 1544 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:55:30.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:30 smithi077 bash[83127]: audit 2024-09-13T00:55:29.504530+0000 mon.a (mon.0) 1545 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:55:30.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:30 smithi077 bash[83127]: audit 2024-09-13T00:55:29.917648+0000 mon.a (mon.0) 1546 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:55:30.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:30 smithi077 bash[83127]: audit 2024-09-13T00:55:29.918919+0000 mon.a (mon.0) 1547 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:55:30.780 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:30 smithi077 bash[83127]: audit 2024-09-13T00:55:29.930229+0000 mon.a (mon.0) 1548 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:55:30.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:30 smithi088 bash[68960]: cluster 2024-09-13T00:55:28.935443+0000 mgr.y (mgr.25304) 590 : cluster [DBG] pgmap v1164: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:30.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:30 smithi088 bash[68960]: audit 2024-09-13T00:55:29.492501+0000 mon.a (mon.0) 1544 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:55:30.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:30 smithi088 bash[68960]: audit 2024-09-13T00:55:29.504530+0000 mon.a (mon.0) 1545 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:55:30.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:30 smithi088 bash[68960]: audit 2024-09-13T00:55:29.917648+0000 mon.a (mon.0) 1546 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:55:30.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:30 smithi088 bash[68960]: audit 2024-09-13T00:55:29.918919+0000 mon.a (mon.0) 1547 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:55:30.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:30 smithi088 bash[68960]: audit 2024-09-13T00:55:29.930229+0000 mon.a (mon.0) 1548 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:55:31.687 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:55:32.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:32 smithi088 bash[68960]: cluster 2024-09-13T00:55:30.936304+0000 mgr.y (mgr.25304) 591 : cluster [DBG] pgmap v1165: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:32.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:55:32 smithi088 bash[61791]: ts=2024-09-13T00:55:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:55:32.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:32 smithi077 bash[80681]: cluster 2024-09-13T00:55:30.936304+0000 mgr.y (mgr.25304) 591 : cluster [DBG] pgmap v1165: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:32.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:32 smithi077 bash[83127]: cluster 2024-09-13T00:55:30.936304+0000 mgr.y (mgr.25304) 591 : cluster [DBG] pgmap v1165: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:34.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:34 smithi077 bash[80681]: cluster 2024-09-13T00:55:32.937088+0000 mgr.y (mgr.25304) 592 : cluster [DBG] pgmap v1166: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:34.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:34 smithi077 bash[83127]: cluster 2024-09-13T00:55:32.937088+0000 mgr.y (mgr.25304) 592 : cluster [DBG] pgmap v1166: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:34.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:34 smithi088 bash[68960]: cluster 2024-09-13T00:55:32.937088+0000 mgr.y (mgr.25304) 592 : cluster [DBG] pgmap v1166: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:35.490 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:55:35.491 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:55:35.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:35 smithi077 bash[80681]: audit 2024-09-13T00:55:35.493048+0000 mon.a (mon.0) 1549 : audit [DBG] from='client.? 172.21.15.77:0/4018901406' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:55:35.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:35 smithi077 bash[83127]: audit 2024-09-13T00:55:35.493048+0000 mon.a (mon.0) 1549 : audit [DBG] from='client.? 172.21.15.77:0/4018901406' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:55:35.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:35 smithi088 bash[68960]: audit 2024-09-13T00:55:35.493048+0000 mon.a (mon.0) 1549 : audit [DBG] from='client.? 172.21.15.77:0/4018901406' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:55:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:36 smithi077 bash[80681]: cluster 2024-09-13T00:55:34.938255+0000 mgr.y (mgr.25304) 593 : cluster [DBG] pgmap v1167: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:36.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:36 smithi077 bash[83127]: cluster 2024-09-13T00:55:34.938255+0000 mgr.y (mgr.25304) 593 : cluster [DBG] pgmap v1167: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:36.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:36 smithi088 bash[68960]: cluster 2024-09-13T00:55:34.938255+0000 mgr.y (mgr.25304) 593 : cluster [DBG] pgmap v1167: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:38.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:38 smithi077 bash[80681]: cluster 2024-09-13T00:55:36.939015+0000 mgr.y (mgr.25304) 594 : cluster [DBG] pgmap v1168: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:38.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:55:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:55:38] "GET /metrics HTTP/1.1" 200 34265 "" "Prometheus/2.43.0" 2024-09-13T00:55:38.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:38 smithi077 bash[83127]: cluster 2024-09-13T00:55:36.939015+0000 mgr.y (mgr.25304) 594 : cluster [DBG] pgmap v1168: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:38.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:38 smithi088 bash[68960]: cluster 2024-09-13T00:55:36.939015+0000 mgr.y (mgr.25304) 594 : cluster [DBG] pgmap v1168: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:39.333 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:55:40.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:40 smithi077 bash[80681]: cluster 2024-09-13T00:55:38.939801+0000 mgr.y (mgr.25304) 595 : cluster [DBG] pgmap v1169: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:40.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:40 smithi077 bash[83127]: cluster 2024-09-13T00:55:38.939801+0000 mgr.y (mgr.25304) 595 : cluster [DBG] pgmap v1169: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:40.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:40 smithi088 bash[68960]: cluster 2024-09-13T00:55:38.939801+0000 mgr.y (mgr.25304) 595 : cluster [DBG] pgmap v1169: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:42 smithi088 bash[68960]: cluster 2024-09-13T00:55:40.940704+0000 mgr.y (mgr.25304) 596 : cluster [DBG] pgmap v1170: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:42.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:55:42 smithi088 bash[61791]: ts=2024-09-13T00:55:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:55:42.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:42 smithi077 bash[80681]: cluster 2024-09-13T00:55:40.940704+0000 mgr.y (mgr.25304) 596 : cluster [DBG] pgmap v1170: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:42.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:42 smithi077 bash[83127]: cluster 2024-09-13T00:55:40.940704+0000 mgr.y (mgr.25304) 596 : cluster [DBG] pgmap v1170: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:43.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:43 smithi077 bash[80681]: audit 2024-09-13T00:55:43.433844+0000 mon.a (mon.0) 1550 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:55:43.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:43 smithi077 bash[83127]: audit 2024-09-13T00:55:43.433844+0000 mon.a (mon.0) 1550 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:55:43.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:43 smithi088 bash[68960]: audit 2024-09-13T00:55:43.433844+0000 mon.a (mon.0) 1550 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:55:44.128 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:55:44.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:44 smithi088 bash[68960]: cluster 2024-09-13T00:55:42.941498+0000 mgr.y (mgr.25304) 597 : cluster [DBG] pgmap v1171: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:45.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:44 smithi077 bash[80681]: cluster 2024-09-13T00:55:42.941498+0000 mgr.y (mgr.25304) 597 : cluster [DBG] pgmap v1171: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:45.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:44 smithi077 bash[83127]: cluster 2024-09-13T00:55:42.941498+0000 mgr.y (mgr.25304) 597 : cluster [DBG] pgmap v1171: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:46.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:46 smithi077 bash[83127]: cluster 2024-09-13T00:55:44.942710+0000 mgr.y (mgr.25304) 598 : cluster [DBG] pgmap v1172: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:46.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:46 smithi077 bash[80681]: cluster 2024-09-13T00:55:44.942710+0000 mgr.y (mgr.25304) 598 : cluster [DBG] pgmap v1172: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:46.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:46 smithi088 bash[68960]: cluster 2024-09-13T00:55:44.942710+0000 mgr.y (mgr.25304) 598 : cluster [DBG] pgmap v1172: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:47.979 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:55:47.979 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:55:48.778 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:48 smithi077 bash[83127]: cluster 2024-09-13T00:55:46.943419+0000 mgr.y (mgr.25304) 599 : cluster [DBG] pgmap v1173: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:48.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:48 smithi077 bash[83127]: audit 2024-09-13T00:55:47.981025+0000 mon.a (mon.0) 1551 : audit [DBG] from='client.? 172.21.15.77:0/1459694946' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:55:48.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:55:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:55:48] "GET /metrics HTTP/1.1" 200 34266 "" "Prometheus/2.43.0" 2024-09-13T00:55:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:48 smithi077 bash[80681]: cluster 2024-09-13T00:55:46.943419+0000 mgr.y (mgr.25304) 599 : cluster [DBG] pgmap v1173: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:48 smithi077 bash[80681]: audit 2024-09-13T00:55:47.981025+0000 mon.a (mon.0) 1551 : audit [DBG] from='client.? 172.21.15.77:0/1459694946' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:55:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:48 smithi088 bash[68960]: cluster 2024-09-13T00:55:46.943419+0000 mgr.y (mgr.25304) 599 : cluster [DBG] pgmap v1173: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:48 smithi088 bash[68960]: audit 2024-09-13T00:55:47.981025+0000 mon.a (mon.0) 1551 : audit [DBG] from='client.? 172.21.15.77:0/1459694946' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:55:50.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:50 smithi088 bash[68960]: cluster 2024-09-13T00:55:48.944231+0000 mgr.y (mgr.25304) 600 : cluster [DBG] pgmap v1174: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:51.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:50 smithi077 bash[80681]: cluster 2024-09-13T00:55:48.944231+0000 mgr.y (mgr.25304) 600 : cluster [DBG] pgmap v1174: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:51.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:50 smithi077 bash[83127]: cluster 2024-09-13T00:55:48.944231+0000 mgr.y (mgr.25304) 600 : cluster [DBG] pgmap v1174: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:51.763 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:55:52.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:52 smithi088 bash[68960]: cluster 2024-09-13T00:55:50.945097+0000 mgr.y (mgr.25304) 601 : cluster [DBG] pgmap v1175: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:52.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:55:52 smithi088 bash[61791]: ts=2024-09-13T00:55:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:55:53.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:52 smithi077 bash[80681]: cluster 2024-09-13T00:55:50.945097+0000 mgr.y (mgr.25304) 601 : cluster [DBG] pgmap v1175: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:53.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:52 smithi077 bash[83127]: cluster 2024-09-13T00:55:50.945097+0000 mgr.y (mgr.25304) 601 : cluster [DBG] pgmap v1175: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:54.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:54 smithi088 bash[68960]: cluster 2024-09-13T00:55:52.945866+0000 mgr.y (mgr.25304) 602 : cluster [DBG] pgmap v1176: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:55.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:54 smithi077 bash[80681]: cluster 2024-09-13T00:55:52.945866+0000 mgr.y (mgr.25304) 602 : cluster [DBG] pgmap v1176: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:55.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:54 smithi077 bash[83127]: cluster 2024-09-13T00:55:52.945866+0000 mgr.y (mgr.25304) 602 : cluster [DBG] pgmap v1176: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:56.548 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:55:56.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:56 smithi088 bash[68960]: cluster 2024-09-13T00:55:54.946821+0000 mgr.y (mgr.25304) 603 : cluster [DBG] pgmap v1177: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:57.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:56 smithi077 bash[80681]: cluster 2024-09-13T00:55:54.946821+0000 mgr.y (mgr.25304) 603 : cluster [DBG] pgmap v1177: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:57.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:56 smithi077 bash[83127]: cluster 2024-09-13T00:55:54.946821+0000 mgr.y (mgr.25304) 603 : cluster [DBG] pgmap v1177: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:58.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:58 smithi077 bash[83127]: cluster 2024-09-13T00:55:56.947578+0000 mgr.y (mgr.25304) 604 : cluster [DBG] pgmap v1178: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:58.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:58 smithi077 bash[83127]: audit 2024-09-13T00:55:58.433960+0000 mon.a (mon.0) 1552 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:55:58.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:58 smithi077 bash[80681]: cluster 2024-09-13T00:55:56.947578+0000 mgr.y (mgr.25304) 604 : cluster [DBG] pgmap v1178: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:58.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:58 smithi077 bash[80681]: audit 2024-09-13T00:55:58.433960+0000 mon.a (mon.0) 1552 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:55:58.780 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:55:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:55:58] "GET /metrics HTTP/1.1" 200 34260 "" "Prometheus/2.43.0" 2024-09-13T00:55:58.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:58 smithi088 bash[68960]: cluster 2024-09-13T00:55:56.947578+0000 mgr.y (mgr.25304) 604 : cluster [DBG] pgmap v1178: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:55:58.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:58 smithi088 bash[68960]: audit 2024-09-13T00:55:58.433960+0000 mon.a (mon.0) 1552 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:56:00.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:55:59 smithi077 bash[80681]: cluster 2024-09-13T00:55:58.948384+0000 mgr.y (mgr.25304) 605 : cluster [DBG] pgmap v1179: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:00.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:55:59 smithi077 bash[83127]: cluster 2024-09-13T00:55:58.948384+0000 mgr.y (mgr.25304) 605 : cluster [DBG] pgmap v1179: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:00.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:55:59 smithi088 bash[68960]: cluster 2024-09-13T00:55:58.948384+0000 mgr.y (mgr.25304) 605 : cluster [DBG] pgmap v1179: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:00.494 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:56:00.495 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:56:00.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:00 smithi077 bash[80681]: audit 2024-09-13T00:56:00.497044+0000 mon.a (mon.0) 1553 : audit [DBG] from='client.? 172.21.15.77:0/4188199243' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:56:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:00 smithi077 bash[83127]: audit 2024-09-13T00:56:00.497044+0000 mon.a (mon.0) 1553 : audit [DBG] from='client.? 172.21.15.77:0/4188199243' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:56:01.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:00 smithi088 bash[68960]: audit 2024-09-13T00:56:00.497044+0000 mon.a (mon.0) 1553 : audit [DBG] from='client.? 172.21.15.77:0/4188199243' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:56:02.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:01 smithi077 bash[83127]: cluster 2024-09-13T00:56:00.949293+0000 mgr.y (mgr.25304) 606 : cluster [DBG] pgmap v1180: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:02.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:01 smithi077 bash[80681]: cluster 2024-09-13T00:56:00.949293+0000 mgr.y (mgr.25304) 606 : cluster [DBG] pgmap v1180: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:02.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:01 smithi088 bash[68960]: cluster 2024-09-13T00:56:00.949293+0000 mgr.y (mgr.25304) 606 : cluster [DBG] pgmap v1180: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:02.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:56:02 smithi088 bash[61791]: ts=2024-09-13T00:56:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:56:04.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:04 smithi077 bash[80681]: cluster 2024-09-13T00:56:02.950056+0000 mgr.y (mgr.25304) 607 : cluster [DBG] pgmap v1181: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:04.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:04 smithi077 bash[83127]: cluster 2024-09-13T00:56:02.950056+0000 mgr.y (mgr.25304) 607 : cluster [DBG] pgmap v1181: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:04.317 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:56:04.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:04 smithi088 bash[68960]: cluster 2024-09-13T00:56:02.950056+0000 mgr.y (mgr.25304) 607 : cluster [DBG] pgmap v1181: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:06.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:06 smithi077 bash[80681]: cluster 2024-09-13T00:56:04.951059+0000 mgr.y (mgr.25304) 608 : cluster [DBG] pgmap v1182: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:06.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:06 smithi077 bash[83127]: cluster 2024-09-13T00:56:04.951059+0000 mgr.y (mgr.25304) 608 : cluster [DBG] pgmap v1182: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:06.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:06 smithi088 bash[68960]: cluster 2024-09-13T00:56:04.951059+0000 mgr.y (mgr.25304) 608 : cluster [DBG] pgmap v1182: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:08.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:08 smithi077 bash[80681]: cluster 2024-09-13T00:56:06.951802+0000 mgr.y (mgr.25304) 609 : cluster [DBG] pgmap v1183: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:08.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:08 smithi077 bash[83127]: cluster 2024-09-13T00:56:06.951802+0000 mgr.y (mgr.25304) 609 : cluster [DBG] pgmap v1183: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:08.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:08 smithi088 bash[68960]: cluster 2024-09-13T00:56:06.951802+0000 mgr.y (mgr.25304) 609 : cluster [DBG] pgmap v1183: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:08.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:56:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:56:08] "GET /metrics HTTP/1.1" 200 34260 "" "Prometheus/2.43.0" 2024-09-13T00:56:09.106 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:56:10.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:10 smithi077 bash[80681]: cluster 2024-09-13T00:56:08.952613+0000 mgr.y (mgr.25304) 610 : cluster [DBG] pgmap v1184: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:10.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:10 smithi077 bash[83127]: cluster 2024-09-13T00:56:08.952613+0000 mgr.y (mgr.25304) 610 : cluster [DBG] pgmap v1184: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:10.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:10 smithi088 bash[68960]: cluster 2024-09-13T00:56:08.952613+0000 mgr.y (mgr.25304) 610 : cluster [DBG] pgmap v1184: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:12.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:12 smithi077 bash[80681]: cluster 2024-09-13T00:56:10.953500+0000 mgr.y (mgr.25304) 611 : cluster [DBG] pgmap v1185: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:12.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:12 smithi077 bash[83127]: cluster 2024-09-13T00:56:10.953500+0000 mgr.y (mgr.25304) 611 : cluster [DBG] pgmap v1185: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:12.314 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:12 smithi088 bash[68960]: cluster 2024-09-13T00:56:10.953500+0000 mgr.y (mgr.25304) 611 : cluster [DBG] pgmap v1185: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:12.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:56:12 smithi088 bash[61791]: ts=2024-09-13T00:56:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:56:12.863 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:56:12.863 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:56:13.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:13 smithi077 bash[80681]: audit 2024-09-13T00:56:12.865253+0000 mon.a (mon.0) 1554 : audit [DBG] from='client.? 172.21.15.77:0/695574588' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:56:13.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:13 smithi077 bash[83127]: audit 2024-09-13T00:56:12.865253+0000 mon.a (mon.0) 1554 : audit [DBG] from='client.? 172.21.15.77:0/695574588' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:56:13.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:13 smithi088 bash[68960]: audit 2024-09-13T00:56:12.865253+0000 mon.a (mon.0) 1554 : audit [DBG] from='client.? 172.21.15.77:0/695574588' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:56:14.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:14 smithi088 bash[68960]: cluster 2024-09-13T00:56:12.953971+0000 mgr.y (mgr.25304) 612 : cluster [DBG] pgmap v1186: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:14.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:14 smithi088 bash[68960]: audit 2024-09-13T00:56:13.434432+0000 mon.a (mon.0) 1555 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:56:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:14 smithi077 bash[80681]: cluster 2024-09-13T00:56:12.953971+0000 mgr.y (mgr.25304) 612 : cluster [DBG] pgmap v1186: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:14 smithi077 bash[80681]: audit 2024-09-13T00:56:13.434432+0000 mon.a (mon.0) 1555 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:56:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:14 smithi077 bash[83127]: cluster 2024-09-13T00:56:12.953971+0000 mgr.y (mgr.25304) 612 : cluster [DBG] pgmap v1186: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:14 smithi077 bash[83127]: audit 2024-09-13T00:56:13.434432+0000 mon.a (mon.0) 1555 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:56:16.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:16 smithi088 bash[68960]: cluster 2024-09-13T00:56:14.954870+0000 mgr.y (mgr.25304) 613 : cluster [DBG] pgmap v1187: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:16.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:16 smithi077 bash[80681]: cluster 2024-09-13T00:56:14.954870+0000 mgr.y (mgr.25304) 613 : cluster [DBG] pgmap v1187: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:16.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:16 smithi077 bash[83127]: cluster 2024-09-13T00:56:14.954870+0000 mgr.y (mgr.25304) 613 : cluster [DBG] pgmap v1187: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:16.688 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:56:18.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:18 smithi088 bash[68960]: cluster 2024-09-13T00:56:16.955622+0000 mgr.y (mgr.25304) 614 : cluster [DBG] pgmap v1188: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:18.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:18 smithi077 bash[80681]: cluster 2024-09-13T00:56:16.955622+0000 mgr.y (mgr.25304) 614 : cluster [DBG] pgmap v1188: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:18.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:18 smithi077 bash[83127]: cluster 2024-09-13T00:56:16.955622+0000 mgr.y (mgr.25304) 614 : cluster [DBG] pgmap v1188: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:18.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:56:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:56:18] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-09-13T00:56:20.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:20 smithi088 bash[68960]: cluster 2024-09-13T00:56:18.956539+0000 mgr.y (mgr.25304) 615 : cluster [DBG] pgmap v1189: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:20.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:20 smithi077 bash[80681]: cluster 2024-09-13T00:56:18.956539+0000 mgr.y (mgr.25304) 615 : cluster [DBG] pgmap v1189: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:20.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:20 smithi077 bash[83127]: cluster 2024-09-13T00:56:18.956539+0000 mgr.y (mgr.25304) 615 : cluster [DBG] pgmap v1189: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:21.490 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:56:22.315 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:22 smithi088 bash[68960]: cluster 2024-09-13T00:56:20.957485+0000 mgr.y (mgr.25304) 616 : cluster [DBG] pgmap v1190: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:22.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:22 smithi077 bash[80681]: cluster 2024-09-13T00:56:20.957485+0000 mgr.y (mgr.25304) 616 : cluster [DBG] pgmap v1190: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:22.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:22 smithi077 bash[83127]: cluster 2024-09-13T00:56:20.957485+0000 mgr.y (mgr.25304) 616 : cluster [DBG] pgmap v1190: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:22.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:56:22 smithi088 bash[61791]: ts=2024-09-13T00:56:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:56:24.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:24 smithi077 bash[80681]: cluster 2024-09-13T00:56:22.958001+0000 mgr.y (mgr.25304) 617 : cluster [DBG] pgmap v1191: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:24.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:24 smithi077 bash[83127]: cluster 2024-09-13T00:56:22.958001+0000 mgr.y (mgr.25304) 617 : cluster [DBG] pgmap v1191: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:24.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:24 smithi088 bash[68960]: cluster 2024-09-13T00:56:22.958001+0000 mgr.y (mgr.25304) 617 : cluster [DBG] pgmap v1191: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:25.472 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:56:25.473 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:56:26.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:26 smithi088 bash[68960]: cluster 2024-09-13T00:56:24.958968+0000 mgr.y (mgr.25304) 618 : cluster [DBG] pgmap v1192: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:26.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:26 smithi088 bash[68960]: audit 2024-09-13T00:56:25.475087+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.77:0/69750697' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:56:26.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:26 smithi077 bash[80681]: cluster 2024-09-13T00:56:24.958968+0000 mgr.y (mgr.25304) 618 : cluster [DBG] pgmap v1192: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:26.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:26 smithi077 bash[80681]: audit 2024-09-13T00:56:25.475087+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.77:0/69750697' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:56:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:26 smithi077 bash[83127]: cluster 2024-09-13T00:56:24.958968+0000 mgr.y (mgr.25304) 618 : cluster [DBG] pgmap v1192: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:26.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:26 smithi077 bash[83127]: audit 2024-09-13T00:56:25.475087+0000 mon.c (mon.2) 57 : audit [DBG] from='client.? 172.21.15.77:0/69750697' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:56:28.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:28 smithi088 bash[68960]: cluster 2024-09-13T00:56:26.959722+0000 mgr.y (mgr.25304) 619 : cluster [DBG] pgmap v1193: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:28.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:28 smithi077 bash[80681]: cluster 2024-09-13T00:56:26.959722+0000 mgr.y (mgr.25304) 619 : cluster [DBG] pgmap v1193: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:28.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:28 smithi077 bash[83127]: cluster 2024-09-13T00:56:26.959722+0000 mgr.y (mgr.25304) 619 : cluster [DBG] pgmap v1193: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:28.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:56:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:56:28] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-09-13T00:56:29.232 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:56:29.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:29 smithi088 bash[68960]: audit 2024-09-13T00:56:28.434740+0000 mon.a (mon.0) 1556 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:56:29.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:29 smithi077 bash[80681]: audit 2024-09-13T00:56:28.434740+0000 mon.a (mon.0) 1556 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:56:29.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:29 smithi077 bash[83127]: audit 2024-09-13T00:56:28.434740+0000 mon.a (mon.0) 1556 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:56:30.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:30 smithi088 bash[68960]: cluster 2024-09-13T00:56:28.960531+0000 mgr.y (mgr.25304) 620 : cluster [DBG] pgmap v1194: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:30.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:30 smithi088 bash[68960]: audit 2024-09-13T00:56:30.061333+0000 mon.a (mon.0) 1557 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:56:30.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:30 smithi077 bash[80681]: cluster 2024-09-13T00:56:28.960531+0000 mgr.y (mgr.25304) 620 : cluster [DBG] pgmap v1194: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:30.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:30 smithi077 bash[80681]: audit 2024-09-13T00:56:30.061333+0000 mon.a (mon.0) 1557 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:56:30.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:30 smithi077 bash[83127]: cluster 2024-09-13T00:56:28.960531+0000 mgr.y (mgr.25304) 620 : cluster [DBG] pgmap v1194: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:30.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:30 smithi077 bash[83127]: audit 2024-09-13T00:56:30.061333+0000 mon.a (mon.0) 1557 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:56:32.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:32 smithi088 bash[68960]: cluster 2024-09-13T00:56:30.961426+0000 mgr.y (mgr.25304) 621 : cluster [DBG] pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:32.353 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:56:32 smithi088 bash[61791]: ts=2024-09-13T00:56:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:56:32.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:32 smithi077 bash[80681]: cluster 2024-09-13T00:56:30.961426+0000 mgr.y (mgr.25304) 621 : cluster [DBG] pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:32.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:32 smithi077 bash[83127]: cluster 2024-09-13T00:56:30.961426+0000 mgr.y (mgr.25304) 621 : cluster [DBG] pgmap v1195: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:34.007 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:56:34.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:34 smithi088 bash[68960]: cluster 2024-09-13T00:56:32.962217+0000 mgr.y (mgr.25304) 622 : cluster [DBG] pgmap v1196: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:34.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:34 smithi077 bash[80681]: cluster 2024-09-13T00:56:32.962217+0000 mgr.y (mgr.25304) 622 : cluster [DBG] pgmap v1196: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:34 smithi077 bash[83127]: cluster 2024-09-13T00:56:32.962217+0000 mgr.y (mgr.25304) 622 : cluster [DBG] pgmap v1196: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:36.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:35 smithi077 bash[83127]: audit 2024-09-13T00:56:34.684876+0000 mon.a (mon.0) 1558 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:56:36.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:35 smithi077 bash[83127]: audit 2024-09-13T00:56:34.691333+0000 mon.a (mon.0) 1559 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:56:36.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:35 smithi077 bash[83127]: cluster 2024-09-13T00:56:34.963040+0000 mgr.y (mgr.25304) 623 : cluster [DBG] pgmap v1197: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:36.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:35 smithi077 bash[80681]: audit 2024-09-13T00:56:34.684876+0000 mon.a (mon.0) 1558 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:56:36.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:35 smithi077 bash[80681]: audit 2024-09-13T00:56:34.691333+0000 mon.a (mon.0) 1559 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:56:36.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:35 smithi077 bash[80681]: cluster 2024-09-13T00:56:34.963040+0000 mgr.y (mgr.25304) 623 : cluster [DBG] pgmap v1197: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:36.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:35 smithi088 bash[68960]: audit 2024-09-13T00:56:34.684876+0000 mon.a (mon.0) 1558 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:56:36.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:35 smithi088 bash[68960]: audit 2024-09-13T00:56:34.691333+0000 mon.a (mon.0) 1559 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:56:36.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:35 smithi088 bash[68960]: cluster 2024-09-13T00:56:34.963040+0000 mgr.y (mgr.25304) 623 : cluster [DBG] pgmap v1197: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:37.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:36 smithi077 bash[80681]: audit 2024-09-13T00:56:35.904042+0000 mon.a (mon.0) 1560 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:56:37.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:36 smithi077 bash[80681]: audit 2024-09-13T00:56:35.914250+0000 mon.a (mon.0) 1561 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:56:37.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:36 smithi077 bash[80681]: audit 2024-09-13T00:56:36.316322+0000 mon.a (mon.0) 1562 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:56:37.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:36 smithi077 bash[80681]: audit 2024-09-13T00:56:36.317716+0000 mon.a (mon.0) 1563 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:56:37.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:36 smithi077 bash[80681]: audit 2024-09-13T00:56:36.326907+0000 mon.a (mon.0) 1564 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:56:37.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:36 smithi077 bash[83127]: audit 2024-09-13T00:56:35.904042+0000 mon.a (mon.0) 1560 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:56:37.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:36 smithi077 bash[83127]: audit 2024-09-13T00:56:35.914250+0000 mon.a (mon.0) 1561 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:56:37.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:36 smithi077 bash[83127]: audit 2024-09-13T00:56:36.316322+0000 mon.a (mon.0) 1562 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:56:37.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:36 smithi077 bash[83127]: audit 2024-09-13T00:56:36.317716+0000 mon.a (mon.0) 1563 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:56:37.280 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:36 smithi077 bash[83127]: audit 2024-09-13T00:56:36.326907+0000 mon.a (mon.0) 1564 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:56:37.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:36 smithi088 bash[68960]: audit 2024-09-13T00:56:35.904042+0000 mon.a (mon.0) 1560 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:56:37.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:36 smithi088 bash[68960]: audit 2024-09-13T00:56:35.914250+0000 mon.a (mon.0) 1561 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:56:37.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:36 smithi088 bash[68960]: audit 2024-09-13T00:56:36.316322+0000 mon.a (mon.0) 1562 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:56:37.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:36 smithi088 bash[68960]: audit 2024-09-13T00:56:36.317716+0000 mon.a (mon.0) 1563 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:56:37.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:36 smithi088 bash[68960]: audit 2024-09-13T00:56:36.326907+0000 mon.a (mon.0) 1564 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:56:38.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:37 smithi077 bash[83127]: cluster 2024-09-13T00:56:36.964005+0000 mgr.y (mgr.25304) 624 : cluster [DBG] pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:38.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:37 smithi077 bash[80681]: cluster 2024-09-13T00:56:36.964005+0000 mgr.y (mgr.25304) 624 : cluster [DBG] pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:38.029 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:56:38.030 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:56:38.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:37 smithi088 bash[68960]: cluster 2024-09-13T00:56:36.964005+0000 mgr.y (mgr.25304) 624 : cluster [DBG] pgmap v1198: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:38.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:56:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:56:38] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-09-13T00:56:39.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:38 smithi077 bash[80681]: audit 2024-09-13T00:56:38.031391+0000 mon.a (mon.0) 1565 : audit [DBG] from='client.? 172.21.15.77:0/4022245824' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:56:39.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:38 smithi077 bash[83127]: audit 2024-09-13T00:56:38.031391+0000 mon.a (mon.0) 1565 : audit [DBG] from='client.? 172.21.15.77:0/4022245824' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:56:39.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:38 smithi088 bash[68960]: audit 2024-09-13T00:56:38.031391+0000 mon.a (mon.0) 1565 : audit [DBG] from='client.? 172.21.15.77:0/4022245824' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:56:40.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:39 smithi077 bash[80681]: cluster 2024-09-13T00:56:38.964888+0000 mgr.y (mgr.25304) 625 : cluster [DBG] pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:40.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:39 smithi077 bash[83127]: cluster 2024-09-13T00:56:38.964888+0000 mgr.y (mgr.25304) 625 : cluster [DBG] pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:40.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:39 smithi088 bash[68960]: cluster 2024-09-13T00:56:38.964888+0000 mgr.y (mgr.25304) 625 : cluster [DBG] pgmap v1199: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:41.788 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:56:42.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:42 smithi077 bash[80681]: cluster 2024-09-13T00:56:40.965768+0000 mgr.y (mgr.25304) 626 : cluster [DBG] pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:42.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:42 smithi077 bash[83127]: cluster 2024-09-13T00:56:40.965768+0000 mgr.y (mgr.25304) 626 : cluster [DBG] pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:42.314 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:42 smithi088 bash[68960]: cluster 2024-09-13T00:56:40.965768+0000 mgr.y (mgr.25304) 626 : cluster [DBG] pgmap v1200: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:42.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:56:42 smithi088 bash[61791]: ts=2024-09-13T00:56:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:56:44.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:44 smithi077 bash[80681]: cluster 2024-09-13T00:56:42.966592+0000 mgr.y (mgr.25304) 627 : cluster [DBG] pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:44.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:44 smithi077 bash[80681]: audit 2024-09-13T00:56:43.435241+0000 mon.a (mon.0) 1566 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:56:44.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:44 smithi077 bash[83127]: cluster 2024-09-13T00:56:42.966592+0000 mgr.y (mgr.25304) 627 : cluster [DBG] pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:44.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:44 smithi077 bash[83127]: audit 2024-09-13T00:56:43.435241+0000 mon.a (mon.0) 1566 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:56:44.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:44 smithi088 bash[68960]: cluster 2024-09-13T00:56:42.966592+0000 mgr.y (mgr.25304) 627 : cluster [DBG] pgmap v1201: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:44.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:44 smithi088 bash[68960]: audit 2024-09-13T00:56:43.435241+0000 mon.a (mon.0) 1566 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:56:46.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:46 smithi077 bash[80681]: cluster 2024-09-13T00:56:44.967543+0000 mgr.y (mgr.25304) 628 : cluster [DBG] pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:46.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:46 smithi077 bash[83127]: cluster 2024-09-13T00:56:44.967543+0000 mgr.y (mgr.25304) 628 : cluster [DBG] pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:46.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:46 smithi088 bash[68960]: cluster 2024-09-13T00:56:44.967543+0000 mgr.y (mgr.25304) 628 : cluster [DBG] pgmap v1202: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:46.575 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:56:48.197 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:48 smithi077 bash[80681]: cluster 2024-09-13T00:56:46.968332+0000 mgr.y (mgr.25304) 629 : cluster [DBG] pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:48.197 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:48 smithi077 bash[83127]: cluster 2024-09-13T00:56:46.968332+0000 mgr.y (mgr.25304) 629 : cluster [DBG] pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:48.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:48 smithi088 bash[68960]: cluster 2024-09-13T00:56:46.968332+0000 mgr.y (mgr.25304) 629 : cluster [DBG] pgmap v1203: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:48.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:56:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:56:48] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-09-13T00:56:50.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:50 smithi077 bash[83127]: cluster 2024-09-13T00:56:48.969242+0000 mgr.y (mgr.25304) 630 : cluster [DBG] pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:50.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:50 smithi077 bash[80681]: cluster 2024-09-13T00:56:48.969242+0000 mgr.y (mgr.25304) 630 : cluster [DBG] pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:50.283 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:56:50.283 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:56:50.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:50 smithi088 bash[68960]: cluster 2024-09-13T00:56:48.969242+0000 mgr.y (mgr.25304) 630 : cluster [DBG] pgmap v1204: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:51.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:51 smithi077 bash[83127]: audit 2024-09-13T00:56:50.285951+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.77:0/2944214436' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:56:51.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:51 smithi077 bash[80681]: audit 2024-09-13T00:56:50.285951+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.77:0/2944214436' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:56:51.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:51 smithi088 bash[68960]: audit 2024-09-13T00:56:50.285951+0000 mon.c (mon.2) 58 : audit [DBG] from='client.? 172.21.15.77:0/2944214436' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:56:52.314 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:52 smithi088 bash[68960]: cluster 2024-09-13T00:56:50.970209+0000 mgr.y (mgr.25304) 631 : cluster [DBG] pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:52.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:52 smithi077 bash[80681]: cluster 2024-09-13T00:56:50.970209+0000 mgr.y (mgr.25304) 631 : cluster [DBG] pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:52.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:52 smithi077 bash[83127]: cluster 2024-09-13T00:56:50.970209+0000 mgr.y (mgr.25304) 631 : cluster [DBG] pgmap v1205: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:52.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:56:52 smithi088 bash[61791]: ts=2024-09-13T00:56:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:56:54.006 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:56:54.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:54 smithi077 bash[80681]: cluster 2024-09-13T00:56:52.971062+0000 mgr.y (mgr.25304) 632 : cluster [DBG] pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:54.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:54 smithi077 bash[83127]: cluster 2024-09-13T00:56:52.971062+0000 mgr.y (mgr.25304) 632 : cluster [DBG] pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:54.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:54 smithi088 bash[68960]: cluster 2024-09-13T00:56:52.971062+0000 mgr.y (mgr.25304) 632 : cluster [DBG] pgmap v1206: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:56.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:56 smithi088 bash[68960]: cluster 2024-09-13T00:56:54.972033+0000 mgr.y (mgr.25304) 633 : cluster [DBG] pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:56.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:56 smithi077 bash[80681]: cluster 2024-09-13T00:56:54.972033+0000 mgr.y (mgr.25304) 633 : cluster [DBG] pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:56 smithi077 bash[83127]: cluster 2024-09-13T00:56:54.972033+0000 mgr.y (mgr.25304) 633 : cluster [DBG] pgmap v1207: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:58.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:58 smithi088 bash[68960]: cluster 2024-09-13T00:56:56.972809+0000 mgr.y (mgr.25304) 634 : cluster [DBG] pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:58.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:58 smithi077 bash[80681]: cluster 2024-09-13T00:56:56.972809+0000 mgr.y (mgr.25304) 634 : cluster [DBG] pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:58.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:58 smithi077 bash[83127]: cluster 2024-09-13T00:56:56.972809+0000 mgr.y (mgr.25304) 634 : cluster [DBG] pgmap v1208: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:56:58.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:56:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:56:58] "GET /metrics HTTP/1.1" 200 34265 "" "Prometheus/2.43.0" 2024-09-13T00:56:58.797 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:56:59.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:56:59 smithi077 bash[83127]: audit 2024-09-13T00:56:58.435870+0000 mon.a (mon.0) 1567 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:56:59.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:56:59 smithi077 bash[80681]: audit 2024-09-13T00:56:58.435870+0000 mon.a (mon.0) 1567 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:56:59.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:56:59 smithi088 bash[68960]: audit 2024-09-13T00:56:58.435870+0000 mon.a (mon.0) 1567 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:57:00.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:00 smithi088 bash[68960]: cluster 2024-09-13T00:56:58.973649+0000 mgr.y (mgr.25304) 635 : cluster [DBG] pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:00.366 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:00 smithi077 bash[83127]: cluster 2024-09-13T00:56:58.973649+0000 mgr.y (mgr.25304) 635 : cluster [DBG] pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:00.366 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:00 smithi077 bash[80681]: cluster 2024-09-13T00:56:58.973649+0000 mgr.y (mgr.25304) 635 : cluster [DBG] pgmap v1209: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:02.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:02 smithi088 bash[68960]: cluster 2024-09-13T00:57:00.974616+0000 mgr.y (mgr.25304) 636 : cluster [DBG] pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:02.353 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:57:02 smithi088 bash[61791]: ts=2024-09-13T00:57:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:57:02.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:02 smithi077 bash[80681]: cluster 2024-09-13T00:57:00.974616+0000 mgr.y (mgr.25304) 636 : cluster [DBG] pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:02.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:02 smithi077 bash[83127]: cluster 2024-09-13T00:57:00.974616+0000 mgr.y (mgr.25304) 636 : cluster [DBG] pgmap v1210: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:02.637 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:57:02.637 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:57:03.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:03 smithi088 bash[68960]: audit 2024-09-13T00:57:02.639144+0000 mon.a (mon.0) 1568 : audit [DBG] from='client.? 172.21.15.77:0/2811526496' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:57:03.375 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:03 smithi077 bash[80681]: audit 2024-09-13T00:57:02.639144+0000 mon.a (mon.0) 1568 : audit [DBG] from='client.? 172.21.15.77:0/2811526496' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:57:03.375 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:03 smithi077 bash[83127]: audit 2024-09-13T00:57:02.639144+0000 mon.a (mon.0) 1568 : audit [DBG] from='client.? 172.21.15.77:0/2811526496' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:57:04.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:04 smithi088 bash[68960]: cluster 2024-09-13T00:57:02.975418+0000 mgr.y (mgr.25304) 637 : cluster [DBG] pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:04.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:04 smithi077 bash[80681]: cluster 2024-09-13T00:57:02.975418+0000 mgr.y (mgr.25304) 637 : cluster [DBG] pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:04.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:04 smithi077 bash[83127]: cluster 2024-09-13T00:57:02.975418+0000 mgr.y (mgr.25304) 637 : cluster [DBG] pgmap v1211: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:06.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:06 smithi088 bash[68960]: cluster 2024-09-13T00:57:04.976350+0000 mgr.y (mgr.25304) 638 : cluster [DBG] pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:06.377 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:57:06.389 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:06 smithi077 bash[80681]: cluster 2024-09-13T00:57:04.976350+0000 mgr.y (mgr.25304) 638 : cluster [DBG] pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:06.389 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:06 smithi077 bash[83127]: cluster 2024-09-13T00:57:04.976350+0000 mgr.y (mgr.25304) 638 : cluster [DBG] pgmap v1212: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:08.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:08 smithi088 bash[68960]: cluster 2024-09-13T00:57:06.977088+0000 mgr.y (mgr.25304) 639 : cluster [DBG] pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:08.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:08 smithi077 bash[80681]: cluster 2024-09-13T00:57:06.977088+0000 mgr.y (mgr.25304) 639 : cluster [DBG] pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:08.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:08 smithi077 bash[83127]: cluster 2024-09-13T00:57:06.977088+0000 mgr.y (mgr.25304) 639 : cluster [DBG] pgmap v1213: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:08.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:57:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:57:08] "GET /metrics HTTP/1.1" 200 34265 "" "Prometheus/2.43.0" 2024-09-13T00:57:10.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:10 smithi088 bash[68960]: cluster 2024-09-13T00:57:08.977917+0000 mgr.y (mgr.25304) 640 : cluster [DBG] pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:10.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:10 smithi077 bash[80681]: cluster 2024-09-13T00:57:08.977917+0000 mgr.y (mgr.25304) 640 : cluster [DBG] pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:10 smithi077 bash[83127]: cluster 2024-09-13T00:57:08.977917+0000 mgr.y (mgr.25304) 640 : cluster [DBG] pgmap v1214: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:11.169 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:57:12.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:12 smithi088 bash[68960]: cluster 2024-09-13T00:57:10.978826+0000 mgr.y (mgr.25304) 641 : cluster [DBG] pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:12.353 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:57:12 smithi088 bash[61791]: ts=2024-09-13T00:57:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:57:12.470 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:12 smithi077 bash[83127]: cluster 2024-09-13T00:57:10.978826+0000 mgr.y (mgr.25304) 641 : cluster [DBG] pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:12.470 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:12 smithi077 bash[80681]: cluster 2024-09-13T00:57:10.978826+0000 mgr.y (mgr.25304) 641 : cluster [DBG] pgmap v1215: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:14 smithi077 bash[80681]: cluster 2024-09-13T00:57:12.979604+0000 mgr.y (mgr.25304) 642 : cluster [DBG] pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:14 smithi077 bash[80681]: audit 2024-09-13T00:57:13.436132+0000 mon.a (mon.0) 1569 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:57:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:14 smithi077 bash[83127]: cluster 2024-09-13T00:57:12.979604+0000 mgr.y (mgr.25304) 642 : cluster [DBG] pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:14.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:14 smithi077 bash[83127]: audit 2024-09-13T00:57:13.436132+0000 mon.a (mon.0) 1569 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:57:14.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:14 smithi088 bash[68960]: cluster 2024-09-13T00:57:12.979604+0000 mgr.y (mgr.25304) 642 : cluster [DBG] pgmap v1216: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:14.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:14 smithi088 bash[68960]: audit 2024-09-13T00:57:13.436132+0000 mon.a (mon.0) 1569 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:57:14.927 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:57:14.927 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:57:15.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:15 smithi077 bash[80681]: audit 2024-09-13T00:57:14.929304+0000 mon.a (mon.0) 1570 : audit [DBG] from='client.? 172.21.15.77:0/2895727272' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:57:15.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:15 smithi077 bash[83127]: audit 2024-09-13T00:57:14.929304+0000 mon.a (mon.0) 1570 : audit [DBG] from='client.? 172.21.15.77:0/2895727272' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:57:15.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:15 smithi088 bash[68960]: audit 2024-09-13T00:57:14.929304+0000 mon.a (mon.0) 1570 : audit [DBG] from='client.? 172.21.15.77:0/2895727272' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:57:16.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:16 smithi077 bash[80681]: cluster 2024-09-13T00:57:14.980110+0000 mgr.y (mgr.25304) 643 : cluster [DBG] pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:16.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:16 smithi077 bash[83127]: cluster 2024-09-13T00:57:14.980110+0000 mgr.y (mgr.25304) 643 : cluster [DBG] pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:16.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:16 smithi088 bash[68960]: cluster 2024-09-13T00:57:14.980110+0000 mgr.y (mgr.25304) 643 : cluster [DBG] pgmap v1217: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:18.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:18 smithi077 bash[80681]: cluster 2024-09-13T00:57:16.980806+0000 mgr.y (mgr.25304) 644 : cluster [DBG] pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:18.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:18 smithi077 bash[83127]: cluster 2024-09-13T00:57:16.980806+0000 mgr.y (mgr.25304) 644 : cluster [DBG] pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:18.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:18 smithi088 bash[68960]: cluster 2024-09-13T00:57:16.980806+0000 mgr.y (mgr.25304) 644 : cluster [DBG] pgmap v1218: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:18.763 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:57:18.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:57:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:57:18] "GET /metrics HTTP/1.1" 200 34262 "" "Prometheus/2.43.0" 2024-09-13T00:57:20.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:20 smithi077 bash[80681]: cluster 2024-09-13T00:57:18.981429+0000 mgr.y (mgr.25304) 645 : cluster [DBG] pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:20.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:20 smithi077 bash[83127]: cluster 2024-09-13T00:57:18.981429+0000 mgr.y (mgr.25304) 645 : cluster [DBG] pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:20.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:20 smithi088 bash[68960]: cluster 2024-09-13T00:57:18.981429+0000 mgr.y (mgr.25304) 645 : cluster [DBG] pgmap v1219: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:22.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:22 smithi077 bash[80681]: cluster 2024-09-13T00:57:20.982295+0000 mgr.y (mgr.25304) 646 : cluster [DBG] pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:22.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:22 smithi077 bash[83127]: cluster 2024-09-13T00:57:20.982295+0000 mgr.y (mgr.25304) 646 : cluster [DBG] pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:22 smithi088 bash[68960]: cluster 2024-09-13T00:57:20.982295+0000 mgr.y (mgr.25304) 646 : cluster [DBG] pgmap v1220: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:22.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:57:22 smithi088 bash[61791]: ts=2024-09-13T00:57:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:57:23.558 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:57:24.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:24 smithi077 bash[80681]: cluster 2024-09-13T00:57:22.983119+0000 mgr.y (mgr.25304) 647 : cluster [DBG] pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:24 smithi077 bash[83127]: cluster 2024-09-13T00:57:22.983119+0000 mgr.y (mgr.25304) 647 : cluster [DBG] pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:24.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:24 smithi088 bash[68960]: cluster 2024-09-13T00:57:22.983119+0000 mgr.y (mgr.25304) 647 : cluster [DBG] pgmap v1221: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:26.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:26 smithi077 bash[80681]: cluster 2024-09-13T00:57:24.984077+0000 mgr.y (mgr.25304) 648 : cluster [DBG] pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:26.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:26 smithi077 bash[83127]: cluster 2024-09-13T00:57:24.984077+0000 mgr.y (mgr.25304) 648 : cluster [DBG] pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:26.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:26 smithi088 bash[68960]: cluster 2024-09-13T00:57:24.984077+0000 mgr.y (mgr.25304) 648 : cluster [DBG] pgmap v1222: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:27.621 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:57:27.622 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:57:28.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:28 smithi077 bash[80681]: cluster 2024-09-13T00:57:26.984853+0000 mgr.y (mgr.25304) 649 : cluster [DBG] pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:28.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:28 smithi077 bash[80681]: audit 2024-09-13T00:57:27.623934+0000 mon.a (mon.0) 1571 : audit [DBG] from='client.? 172.21.15.77:0/3589273123' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:57:28.527 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:28 smithi077 bash[83127]: cluster 2024-09-13T00:57:26.984853+0000 mgr.y (mgr.25304) 649 : cluster [DBG] pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:28.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:28 smithi077 bash[83127]: audit 2024-09-13T00:57:27.623934+0000 mon.a (mon.0) 1571 : audit [DBG] from='client.? 172.21.15.77:0/3589273123' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:57:28.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:28 smithi088 bash[68960]: cluster 2024-09-13T00:57:26.984853+0000 mgr.y (mgr.25304) 649 : cluster [DBG] pgmap v1223: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:28.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:28 smithi088 bash[68960]: audit 2024-09-13T00:57:27.623934+0000 mon.a (mon.0) 1571 : audit [DBG] from='client.? 172.21.15.77:0/3589273123' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:57:28.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:57:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:57:28] "GET /metrics HTTP/1.1" 200 34264 "" "Prometheus/2.43.0" 2024-09-13T00:57:29.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:29 smithi077 bash[80681]: audit 2024-09-13T00:57:28.436628+0000 mon.a (mon.0) 1572 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:57:29.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:29 smithi077 bash[83127]: audit 2024-09-13T00:57:28.436628+0000 mon.a (mon.0) 1572 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:57:29.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:29 smithi088 bash[68960]: audit 2024-09-13T00:57:28.436628+0000 mon.a (mon.0) 1572 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:57:30.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:30 smithi077 bash[80681]: cluster 2024-09-13T00:57:28.985656+0000 mgr.y (mgr.25304) 650 : cluster [DBG] pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:30.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:30 smithi077 bash[83127]: cluster 2024-09-13T00:57:28.985656+0000 mgr.y (mgr.25304) 650 : cluster [DBG] pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:30.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:30 smithi088 bash[68960]: cluster 2024-09-13T00:57:28.985656+0000 mgr.y (mgr.25304) 650 : cluster [DBG] pgmap v1224: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:31.327 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:57:32.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:32 smithi077 bash[80681]: cluster 2024-09-13T00:57:30.986598+0000 mgr.y (mgr.25304) 651 : cluster [DBG] pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:32.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:32 smithi077 bash[83127]: cluster 2024-09-13T00:57:30.986598+0000 mgr.y (mgr.25304) 651 : cluster [DBG] pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:32.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:32 smithi088 bash[68960]: cluster 2024-09-13T00:57:30.986598+0000 mgr.y (mgr.25304) 651 : cluster [DBG] pgmap v1225: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:32.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:57:32 smithi088 bash[61791]: ts=2024-09-13T00:57:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:57:34.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:34 smithi088 bash[68960]: cluster 2024-09-13T00:57:32.987494+0000 mgr.y (mgr.25304) 652 : cluster [DBG] pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:34.353 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 00:57:33 smithi088 bash[60519]: logger=cleanup t=2024-09-13T00:57:33.966143759Z level=info msg="Completed cleanup jobs" duration=13.301793ms 2024-09-13T00:57:34.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:34 smithi077 bash[80681]: cluster 2024-09-13T00:57:32.987494+0000 mgr.y (mgr.25304) 652 : cluster [DBG] pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:34.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:34 smithi077 bash[83127]: cluster 2024-09-13T00:57:32.987494+0000 mgr.y (mgr.25304) 652 : cluster [DBG] pgmap v1226: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:36.105 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:57:36.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:36 smithi077 bash[80681]: cluster 2024-09-13T00:57:34.988437+0000 mgr.y (mgr.25304) 653 : cluster [DBG] pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:36.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:36 smithi077 bash[83127]: cluster 2024-09-13T00:57:34.988437+0000 mgr.y (mgr.25304) 653 : cluster [DBG] pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:36.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:36 smithi088 bash[68960]: cluster 2024-09-13T00:57:34.988437+0000 mgr.y (mgr.25304) 653 : cluster [DBG] pgmap v1227: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:37.441 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:37 smithi077 bash[80681]: audit 2024-09-13T00:57:36.459683+0000 mon.a (mon.0) 1573 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:57:37.441 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:37 smithi077 bash[83127]: audit 2024-09-13T00:57:36.459683+0000 mon.a (mon.0) 1573 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:57:37.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:37 smithi088 bash[68960]: audit 2024-09-13T00:57:36.459683+0000 mon.a (mon.0) 1573 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:57:38.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:38 smithi077 bash[80681]: cluster 2024-09-13T00:57:36.989207+0000 mgr.y (mgr.25304) 654 : cluster [DBG] pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:38.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:38 smithi077 bash[83127]: cluster 2024-09-13T00:57:36.989207+0000 mgr.y (mgr.25304) 654 : cluster [DBG] pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:38.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:38 smithi088 bash[68960]: cluster 2024-09-13T00:57:36.989207+0000 mgr.y (mgr.25304) 654 : cluster [DBG] pgmap v1228: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:38.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:57:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:57:38] "GET /metrics HTTP/1.1" 200 34264 "" "Prometheus/2.43.0" 2024-09-13T00:57:40.008 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:57:40.009 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:57:40.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:40 smithi077 bash[80681]: cluster 2024-09-13T00:57:38.990038+0000 mgr.y (mgr.25304) 655 : cluster [DBG] pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:40.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:40 smithi077 bash[80681]: audit 2024-09-13T00:57:40.011192+0000 mon.a (mon.0) 1574 : audit [DBG] from='client.? 172.21.15.77:0/1707155032' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:57:40.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:40 smithi077 bash[83127]: cluster 2024-09-13T00:57:38.990038+0000 mgr.y (mgr.25304) 655 : cluster [DBG] pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:40.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:40 smithi077 bash[83127]: audit 2024-09-13T00:57:40.011192+0000 mon.a (mon.0) 1574 : audit [DBG] from='client.? 172.21.15.77:0/1707155032' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:57:40.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:40 smithi088 bash[68960]: cluster 2024-09-13T00:57:38.990038+0000 mgr.y (mgr.25304) 655 : cluster [DBG] pgmap v1229: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:40.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:40 smithi088 bash[68960]: audit 2024-09-13T00:57:40.011192+0000 mon.a (mon.0) 1574 : audit [DBG] from='client.? 172.21.15.77:0/1707155032' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:57:42.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:42 smithi077 bash[80681]: cluster 2024-09-13T00:57:40.991002+0000 mgr.y (mgr.25304) 656 : cluster [DBG] pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:42.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:42 smithi077 bash[80681]: audit 2024-09-13T00:57:42.081233+0000 mon.a (mon.0) 1575 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:57:42.532 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:42 smithi077 bash[80681]: audit 2024-09-13T00:57:42.089600+0000 mon.a (mon.0) 1576 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:57:42.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:42 smithi077 bash[83127]: cluster 2024-09-13T00:57:40.991002+0000 mgr.y (mgr.25304) 656 : cluster [DBG] pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:42.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:42 smithi077 bash[83127]: audit 2024-09-13T00:57:42.081233+0000 mon.a (mon.0) 1575 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:57:42.532 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:42 smithi077 bash[83127]: audit 2024-09-13T00:57:42.089600+0000 mon.a (mon.0) 1576 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:57:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:42 smithi088 bash[68960]: cluster 2024-09-13T00:57:40.991002+0000 mgr.y (mgr.25304) 656 : cluster [DBG] pgmap v1230: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:42.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:42 smithi088 bash[68960]: audit 2024-09-13T00:57:42.081233+0000 mon.a (mon.0) 1575 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:57:42.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:42 smithi088 bash[68960]: audit 2024-09-13T00:57:42.089600+0000 mon.a (mon.0) 1576 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:57:42.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:57:42 smithi088 bash[61791]: ts=2024-09-13T00:57:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:57:43.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:43 smithi088 bash[68960]: audit 2024-09-13T00:57:42.340347+0000 mon.a (mon.0) 1577 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:57:43.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:43 smithi088 bash[68960]: audit 2024-09-13T00:57:42.352062+0000 mon.a (mon.0) 1578 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:57:43.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:43 smithi088 bash[68960]: audit 2024-09-13T00:57:42.780637+0000 mon.a (mon.0) 1579 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:57:43.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:43 smithi088 bash[68960]: audit 2024-09-13T00:57:42.782178+0000 mon.a (mon.0) 1580 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:57:43.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:43 smithi088 bash[68960]: audit 2024-09-13T00:57:42.795933+0000 mon.a (mon.0) 1581 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:57:43.761 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:57:43.772 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:43 smithi077 bash[80681]: audit 2024-09-13T00:57:42.340347+0000 mon.a (mon.0) 1577 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:57:43.772 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:43 smithi077 bash[80681]: audit 2024-09-13T00:57:42.352062+0000 mon.a (mon.0) 1578 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:57:43.773 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:43 smithi077 bash[80681]: audit 2024-09-13T00:57:42.780637+0000 mon.a (mon.0) 1579 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:57:43.773 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:43 smithi077 bash[80681]: audit 2024-09-13T00:57:42.782178+0000 mon.a (mon.0) 1580 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:57:43.773 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:43 smithi077 bash[80681]: audit 2024-09-13T00:57:42.795933+0000 mon.a (mon.0) 1581 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:57:43.773 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:43 smithi077 bash[83127]: audit 2024-09-13T00:57:42.340347+0000 mon.a (mon.0) 1577 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:57:43.773 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:43 smithi077 bash[83127]: audit 2024-09-13T00:57:42.352062+0000 mon.a (mon.0) 1578 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:57:43.773 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:43 smithi077 bash[83127]: audit 2024-09-13T00:57:42.780637+0000 mon.a (mon.0) 1579 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:57:43.773 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:43 smithi077 bash[83127]: audit 2024-09-13T00:57:42.782178+0000 mon.a (mon.0) 1580 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:57:43.773 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:43 smithi077 bash[83127]: audit 2024-09-13T00:57:42.795933+0000 mon.a (mon.0) 1581 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:57:44.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:44 smithi088 bash[68960]: cluster 2024-09-13T00:57:42.991913+0000 mgr.y (mgr.25304) 657 : cluster [DBG] pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:44.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:44 smithi088 bash[68960]: audit 2024-09-13T00:57:43.437077+0000 mon.a (mon.0) 1582 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:57:44.780 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:44 smithi077 bash[80681]: cluster 2024-09-13T00:57:42.991913+0000 mgr.y (mgr.25304) 657 : cluster [DBG] pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:44.865 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:44 smithi077 bash[80681]: audit 2024-09-13T00:57:43.437077+0000 mon.a (mon.0) 1582 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:57:44.865 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:44 smithi077 bash[83127]: cluster 2024-09-13T00:57:42.991913+0000 mgr.y (mgr.25304) 657 : cluster [DBG] pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:44.866 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:44 smithi077 bash[83127]: audit 2024-09-13T00:57:43.437077+0000 mon.a (mon.0) 1582 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:57:46.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:46 smithi088 bash[68960]: cluster 2024-09-13T00:57:44.992789+0000 mgr.y (mgr.25304) 658 : cluster [DBG] pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:46.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:46 smithi077 bash[80681]: cluster 2024-09-13T00:57:44.992789+0000 mgr.y (mgr.25304) 658 : cluster [DBG] pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:46.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:46 smithi077 bash[83127]: cluster 2024-09-13T00:57:44.992789+0000 mgr.y (mgr.25304) 658 : cluster [DBG] pgmap v1232: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:48.551 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:57:48.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:48 smithi088 bash[68960]: cluster 2024-09-13T00:57:46.993526+0000 mgr.y (mgr.25304) 659 : cluster [DBG] pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:48.647 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:57:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:57:48] "GET /metrics HTTP/1.1" 200 34266 "" "Prometheus/2.43.0" 2024-09-13T00:57:48.648 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:48 smithi077 bash[80681]: cluster 2024-09-13T00:57:46.993526+0000 mgr.y (mgr.25304) 659 : cluster [DBG] pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:48.648 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:48 smithi077 bash[83127]: cluster 2024-09-13T00:57:46.993526+0000 mgr.y (mgr.25304) 659 : cluster [DBG] pgmap v1233: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:50.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:50 smithi088 bash[68960]: cluster 2024-09-13T00:57:48.994320+0000 mgr.y (mgr.25304) 660 : cluster [DBG] pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:50.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:50 smithi077 bash[80681]: cluster 2024-09-13T00:57:48.994320+0000 mgr.y (mgr.25304) 660 : cluster [DBG] pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:50.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:50 smithi077 bash[83127]: cluster 2024-09-13T00:57:48.994320+0000 mgr.y (mgr.25304) 660 : cluster [DBG] pgmap v1234: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:52.477 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:57:52.477 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:57:52.489 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:52 smithi077 bash[80681]: cluster 2024-09-13T00:57:50.995262+0000 mgr.y (mgr.25304) 661 : cluster [DBG] pgmap v1235: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:52.489 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:52 smithi077 bash[83127]: cluster 2024-09-13T00:57:50.995262+0000 mgr.y (mgr.25304) 661 : cluster [DBG] pgmap v1235: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:52.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:52 smithi088 bash[68960]: cluster 2024-09-13T00:57:50.995262+0000 mgr.y (mgr.25304) 661 : cluster [DBG] pgmap v1235: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:52.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:57:52 smithi088 bash[61791]: ts=2024-09-13T00:57:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:57:53.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:53 smithi077 bash[80681]: audit 2024-09-13T00:57:52.479708+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.77:0/2125424551' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:57:53.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:53 smithi077 bash[83127]: audit 2024-09-13T00:57:52.479708+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.77:0/2125424551' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:57:53.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:53 smithi088 bash[68960]: audit 2024-09-13T00:57:52.479708+0000 mon.c (mon.2) 59 : audit [DBG] from='client.? 172.21.15.77:0/2125424551' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:57:54.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:54 smithi077 bash[80681]: cluster 2024-09-13T00:57:52.995992+0000 mgr.y (mgr.25304) 662 : cluster [DBG] pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:54.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:54 smithi077 bash[83127]: cluster 2024-09-13T00:57:52.995992+0000 mgr.y (mgr.25304) 662 : cluster [DBG] pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:54.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:54 smithi088 bash[68960]: cluster 2024-09-13T00:57:52.995992+0000 mgr.y (mgr.25304) 662 : cluster [DBG] pgmap v1236: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:56.253 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:57:56.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:56 smithi077 bash[80681]: cluster 2024-09-13T00:57:54.996933+0000 mgr.y (mgr.25304) 663 : cluster [DBG] pgmap v1237: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:56 smithi077 bash[83127]: cluster 2024-09-13T00:57:54.996933+0000 mgr.y (mgr.25304) 663 : cluster [DBG] pgmap v1237: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:56.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:56 smithi088 bash[68960]: cluster 2024-09-13T00:57:54.996933+0000 mgr.y (mgr.25304) 663 : cluster [DBG] pgmap v1237: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:58.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:58 smithi077 bash[80681]: cluster 2024-09-13T00:57:56.997681+0000 mgr.y (mgr.25304) 664 : cluster [DBG] pgmap v1238: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:58.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:57:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:57:58] "GET /metrics HTTP/1.1" 200 34262 "" "Prometheus/2.43.0" 2024-09-13T00:57:58.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:58 smithi077 bash[83127]: cluster 2024-09-13T00:57:56.997681+0000 mgr.y (mgr.25304) 664 : cluster [DBG] pgmap v1238: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:58.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:58 smithi088 bash[68960]: cluster 2024-09-13T00:57:56.997681+0000 mgr.y (mgr.25304) 664 : cluster [DBG] pgmap v1238: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:57:59.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:57:59 smithi077 bash[80681]: audit 2024-09-13T00:57:58.437512+0000 mon.a (mon.0) 1583 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:57:59.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:57:59 smithi077 bash[83127]: audit 2024-09-13T00:57:58.437512+0000 mon.a (mon.0) 1583 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:57:59.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:57:59 smithi088 bash[68960]: audit 2024-09-13T00:57:58.437512+0000 mon.a (mon.0) 1583 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:58:00.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:00 smithi077 bash[80681]: cluster 2024-09-13T00:57:58.998197+0000 mgr.y (mgr.25304) 665 : cluster [DBG] pgmap v1239: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:00.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:00 smithi077 bash[83127]: cluster 2024-09-13T00:57:58.998197+0000 mgr.y (mgr.25304) 665 : cluster [DBG] pgmap v1239: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:00.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:00 smithi088 bash[68960]: cluster 2024-09-13T00:57:58.998197+0000 mgr.y (mgr.25304) 665 : cluster [DBG] pgmap v1239: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:01.040 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:58:02.595 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:02 smithi077 bash[80681]: cluster 2024-09-13T00:58:00.999069+0000 mgr.y (mgr.25304) 666 : cluster [DBG] pgmap v1240: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:02.595 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:02 smithi077 bash[83127]: cluster 2024-09-13T00:58:00.999069+0000 mgr.y (mgr.25304) 666 : cluster [DBG] pgmap v1240: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:02.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:02 smithi088 bash[68960]: cluster 2024-09-13T00:58:00.999069+0000 mgr.y (mgr.25304) 666 : cluster [DBG] pgmap v1240: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:02.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:58:02 smithi088 bash[61791]: ts=2024-09-13T00:58:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:58:04.743 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:58:04.743 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:58:04.754 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:04 smithi077 bash[80681]: cluster 2024-09-13T00:58:02.999882+0000 mgr.y (mgr.25304) 667 : cluster [DBG] pgmap v1241: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:04.755 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:04 smithi077 bash[83127]: cluster 2024-09-13T00:58:02.999882+0000 mgr.y (mgr.25304) 667 : cluster [DBG] pgmap v1241: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:04.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:04 smithi088 bash[68960]: cluster 2024-09-13T00:58:02.999882+0000 mgr.y (mgr.25304) 667 : cluster [DBG] pgmap v1241: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:05.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:05 smithi077 bash[80681]: audit 2024-09-13T00:58:04.745275+0000 mon.a (mon.0) 1584 : audit [DBG] from='client.? 172.21.15.77:0/2309953652' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:58:05.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:05 smithi077 bash[83127]: audit 2024-09-13T00:58:04.745275+0000 mon.a (mon.0) 1584 : audit [DBG] from='client.? 172.21.15.77:0/2309953652' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:58:05.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:05 smithi088 bash[68960]: audit 2024-09-13T00:58:04.745275+0000 mon.a (mon.0) 1584 : audit [DBG] from='client.? 172.21.15.77:0/2309953652' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:58:06.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:06 smithi077 bash[80681]: cluster 2024-09-13T00:58:05.000873+0000 mgr.y (mgr.25304) 668 : cluster [DBG] pgmap v1242: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:06.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:06 smithi077 bash[83127]: cluster 2024-09-13T00:58:05.000873+0000 mgr.y (mgr.25304) 668 : cluster [DBG] pgmap v1242: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:06.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:06 smithi088 bash[68960]: cluster 2024-09-13T00:58:05.000873+0000 mgr.y (mgr.25304) 668 : cluster [DBG] pgmap v1242: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:08.546 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:58:08.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:08 smithi077 bash[80681]: cluster 2024-09-13T00:58:07.001641+0000 mgr.y (mgr.25304) 669 : cluster [DBG] pgmap v1243: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:08.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:08 smithi077 bash[83127]: cluster 2024-09-13T00:58:07.001641+0000 mgr.y (mgr.25304) 669 : cluster [DBG] pgmap v1243: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:08.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:58:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:58:08] "GET /metrics HTTP/1.1" 200 34262 "" "Prometheus/2.43.0" 2024-09-13T00:58:08.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:08 smithi088 bash[68960]: cluster 2024-09-13T00:58:07.001641+0000 mgr.y (mgr.25304) 669 : cluster [DBG] pgmap v1243: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:10.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:10 smithi077 bash[80681]: cluster 2024-09-13T00:58:09.002471+0000 mgr.y (mgr.25304) 670 : cluster [DBG] pgmap v1244: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:10.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:10 smithi077 bash[83127]: cluster 2024-09-13T00:58:09.002471+0000 mgr.y (mgr.25304) 670 : cluster [DBG] pgmap v1244: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:10.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:10 smithi088 bash[68960]: cluster 2024-09-13T00:58:09.002471+0000 mgr.y (mgr.25304) 670 : cluster [DBG] pgmap v1244: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:12.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:12 smithi088 bash[68960]: cluster 2024-09-13T00:58:11.003417+0000 mgr.y (mgr.25304) 671 : cluster [DBG] pgmap v1245: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:12.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:58:12 smithi088 bash[61791]: ts=2024-09-13T00:58:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:58:12.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:12 smithi077 bash[80681]: cluster 2024-09-13T00:58:11.003417+0000 mgr.y (mgr.25304) 671 : cluster [DBG] pgmap v1245: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:12.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:12 smithi077 bash[83127]: cluster 2024-09-13T00:58:11.003417+0000 mgr.y (mgr.25304) 671 : cluster [DBG] pgmap v1245: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:13.331 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:58:14.695 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:14 smithi077 bash[80681]: cluster 2024-09-13T00:58:13.004217+0000 mgr.y (mgr.25304) 672 : cluster [DBG] pgmap v1246: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:14.695 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:14 smithi077 bash[80681]: audit 2024-09-13T00:58:13.442329+0000 mon.a (mon.0) 1585 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:58:14.696 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:14 smithi077 bash[83127]: cluster 2024-09-13T00:58:13.004217+0000 mgr.y (mgr.25304) 672 : cluster [DBG] pgmap v1246: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:14.696 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:14 smithi077 bash[83127]: audit 2024-09-13T00:58:13.442329+0000 mon.a (mon.0) 1585 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:58:14.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:14 smithi088 bash[68960]: cluster 2024-09-13T00:58:13.004217+0000 mgr.y (mgr.25304) 672 : cluster [DBG] pgmap v1246: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:14.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:14 smithi088 bash[68960]: audit 2024-09-13T00:58:13.442329+0000 mon.a (mon.0) 1585 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:58:16.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:16 smithi077 bash[80681]: cluster 2024-09-13T00:58:15.005200+0000 mgr.y (mgr.25304) 673 : cluster [DBG] pgmap v1247: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:16.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:16 smithi077 bash[83127]: cluster 2024-09-13T00:58:15.005200+0000 mgr.y (mgr.25304) 673 : cluster [DBG] pgmap v1247: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:16.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:16 smithi088 bash[68960]: cluster 2024-09-13T00:58:15.005200+0000 mgr.y (mgr.25304) 673 : cluster [DBG] pgmap v1247: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:17.136 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:58:17.136 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:58:17.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:17 smithi077 bash[80681]: audit 2024-09-13T00:58:17.138567+0000 mon.a (mon.0) 1586 : audit [DBG] from='client.? 172.21.15.77:0/4060013995' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:58:17.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:17 smithi077 bash[83127]: audit 2024-09-13T00:58:17.138567+0000 mon.a (mon.0) 1586 : audit [DBG] from='client.? 172.21.15.77:0/4060013995' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:58:17.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:17 smithi088 bash[68960]: audit 2024-09-13T00:58:17.138567+0000 mon.a (mon.0) 1586 : audit [DBG] from='client.? 172.21.15.77:0/4060013995' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:58:18.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:18 smithi077 bash[80681]: cluster 2024-09-13T00:58:17.005727+0000 mgr.y (mgr.25304) 674 : cluster [DBG] pgmap v1248: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:18.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:58:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:58:18] "GET /metrics HTTP/1.1" 200 34262 "" "Prometheus/2.43.0" 2024-09-13T00:58:18.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:18 smithi077 bash[83127]: cluster 2024-09-13T00:58:17.005727+0000 mgr.y (mgr.25304) 674 : cluster [DBG] pgmap v1248: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:18.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:18 smithi088 bash[68960]: cluster 2024-09-13T00:58:17.005727+0000 mgr.y (mgr.25304) 674 : cluster [DBG] pgmap v1248: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:20.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:20 smithi077 bash[80681]: cluster 2024-09-13T00:58:19.006547+0000 mgr.y (mgr.25304) 675 : cluster [DBG] pgmap v1249: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:20.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:20 smithi077 bash[83127]: cluster 2024-09-13T00:58:19.006547+0000 mgr.y (mgr.25304) 675 : cluster [DBG] pgmap v1249: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:20.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:20 smithi088 bash[68960]: cluster 2024-09-13T00:58:19.006547+0000 mgr.y (mgr.25304) 675 : cluster [DBG] pgmap v1249: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:20.911 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:58:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:22 smithi088 bash[68960]: cluster 2024-09-13T00:58:21.007392+0000 mgr.y (mgr.25304) 676 : cluster [DBG] pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:22.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:58:22 smithi088 bash[61791]: ts=2024-09-13T00:58:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:58:22.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:22 smithi077 bash[80681]: cluster 2024-09-13T00:58:21.007392+0000 mgr.y (mgr.25304) 676 : cluster [DBG] pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:22.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:22 smithi077 bash[83127]: cluster 2024-09-13T00:58:21.007392+0000 mgr.y (mgr.25304) 676 : cluster [DBG] pgmap v1250: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:24.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:24 smithi077 bash[80681]: cluster 2024-09-13T00:58:23.008183+0000 mgr.y (mgr.25304) 677 : cluster [DBG] pgmap v1251: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:24.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:24 smithi077 bash[83127]: cluster 2024-09-13T00:58:23.008183+0000 mgr.y (mgr.25304) 677 : cluster [DBG] pgmap v1251: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:24.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:24 smithi088 bash[68960]: cluster 2024-09-13T00:58:23.008183+0000 mgr.y (mgr.25304) 677 : cluster [DBG] pgmap v1251: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:25.702 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:58:26.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:26 smithi077 bash[80681]: cluster 2024-09-13T00:58:25.009127+0000 mgr.y (mgr.25304) 678 : cluster [DBG] pgmap v1252: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:26.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:26 smithi077 bash[83127]: cluster 2024-09-13T00:58:25.009127+0000 mgr.y (mgr.25304) 678 : cluster [DBG] pgmap v1252: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:26.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:26 smithi088 bash[68960]: cluster 2024-09-13T00:58:25.009127+0000 mgr.y (mgr.25304) 678 : cluster [DBG] pgmap v1252: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:28.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:28 smithi077 bash[80681]: cluster 2024-09-13T00:58:27.009603+0000 mgr.y (mgr.25304) 679 : cluster [DBG] pgmap v1253: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:28.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:28 smithi077 bash[80681]: audit 2024-09-13T00:58:28.438607+0000 mon.a (mon.0) 1587 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:58:28.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:58:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:58:28] "GET /metrics HTTP/1.1" 200 34266 "" "Prometheus/2.43.0" 2024-09-13T00:58:28.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:28 smithi077 bash[83127]: cluster 2024-09-13T00:58:27.009603+0000 mgr.y (mgr.25304) 679 : cluster [DBG] pgmap v1253: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:28.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:28 smithi077 bash[83127]: audit 2024-09-13T00:58:28.438607+0000 mon.a (mon.0) 1587 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:58:28.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:28 smithi088 bash[68960]: cluster 2024-09-13T00:58:27.009603+0000 mgr.y (mgr.25304) 679 : cluster [DBG] pgmap v1253: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:28.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:28 smithi088 bash[68960]: audit 2024-09-13T00:58:28.438607+0000 mon.a (mon.0) 1587 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:58:29.326 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:58:29.326 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:58:29.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:29 smithi077 bash[80681]: audit 2024-09-13T00:58:29.328352+0000 mon.a (mon.0) 1588 : audit [DBG] from='client.? 172.21.15.77:0/309719701' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:58:29.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:29 smithi077 bash[83127]: audit 2024-09-13T00:58:29.328352+0000 mon.a (mon.0) 1588 : audit [DBG] from='client.? 172.21.15.77:0/309719701' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:58:29.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:29 smithi088 bash[68960]: audit 2024-09-13T00:58:29.328352+0000 mon.a (mon.0) 1588 : audit [DBG] from='client.? 172.21.15.77:0/309719701' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:58:30.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:30 smithi077 bash[80681]: cluster 2024-09-13T00:58:29.010341+0000 mgr.y (mgr.25304) 680 : cluster [DBG] pgmap v1254: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:30.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:30 smithi077 bash[83127]: cluster 2024-09-13T00:58:29.010341+0000 mgr.y (mgr.25304) 680 : cluster [DBG] pgmap v1254: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:30.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:30 smithi088 bash[68960]: cluster 2024-09-13T00:58:29.010341+0000 mgr.y (mgr.25304) 680 : cluster [DBG] pgmap v1254: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:32.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:32 smithi088 bash[68960]: cluster 2024-09-13T00:58:31.011195+0000 mgr.y (mgr.25304) 681 : cluster [DBG] pgmap v1255: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:32.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:58:32 smithi088 bash[61791]: ts=2024-09-13T00:58:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:58:32.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:32 smithi077 bash[80681]: cluster 2024-09-13T00:58:31.011195+0000 mgr.y (mgr.25304) 681 : cluster [DBG] pgmap v1255: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:32.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:32 smithi077 bash[83127]: cluster 2024-09-13T00:58:31.011195+0000 mgr.y (mgr.25304) 681 : cluster [DBG] pgmap v1255: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:33.179 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:58:34.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:34 smithi077 bash[80681]: cluster 2024-09-13T00:58:33.012007+0000 mgr.y (mgr.25304) 682 : cluster [DBG] pgmap v1256: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:34.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:34 smithi077 bash[83127]: cluster 2024-09-13T00:58:33.012007+0000 mgr.y (mgr.25304) 682 : cluster [DBG] pgmap v1256: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:34.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:34 smithi088 bash[68960]: cluster 2024-09-13T00:58:33.012007+0000 mgr.y (mgr.25304) 682 : cluster [DBG] pgmap v1256: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:36.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:36 smithi077 bash[80681]: cluster 2024-09-13T00:58:35.012948+0000 mgr.y (mgr.25304) 683 : cluster [DBG] pgmap v1257: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:36.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:36 smithi077 bash[83127]: cluster 2024-09-13T00:58:35.012948+0000 mgr.y (mgr.25304) 683 : cluster [DBG] pgmap v1257: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:36.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:36 smithi088 bash[68960]: cluster 2024-09-13T00:58:35.012948+0000 mgr.y (mgr.25304) 683 : cluster [DBG] pgmap v1257: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:37.964 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:58:38.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:38 smithi077 bash[80681]: cluster 2024-09-13T00:58:37.013680+0000 mgr.y (mgr.25304) 684 : cluster [DBG] pgmap v1258: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:38.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:58:38 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:58:38] "GET /metrics HTTP/1.1" 200 34266 "" "Prometheus/2.43.0" 2024-09-13T00:58:38.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:38 smithi077 bash[83127]: cluster 2024-09-13T00:58:37.013680+0000 mgr.y (mgr.25304) 684 : cluster [DBG] pgmap v1258: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:38.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:38 smithi088 bash[68960]: cluster 2024-09-13T00:58:37.013680+0000 mgr.y (mgr.25304) 684 : cluster [DBG] pgmap v1258: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:40.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:40 smithi077 bash[80681]: cluster 2024-09-13T00:58:39.014602+0000 mgr.y (mgr.25304) 685 : cluster [DBG] pgmap v1259: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:40.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:40 smithi077 bash[83127]: cluster 2024-09-13T00:58:39.014602+0000 mgr.y (mgr.25304) 685 : cluster [DBG] pgmap v1259: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:40.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:40 smithi088 bash[68960]: cluster 2024-09-13T00:58:39.014602+0000 mgr.y (mgr.25304) 685 : cluster [DBG] pgmap v1259: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:41.628 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:58:41.628 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:58:42.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:42 smithi088 bash[68960]: cluster 2024-09-13T00:58:41.015544+0000 mgr.y (mgr.25304) 686 : cluster [DBG] pgmap v1260: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:42.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:42 smithi088 bash[68960]: audit 2024-09-13T00:58:41.630915+0000 mon.a (mon.0) 1589 : audit [DBG] from='client.? 172.21.15.77:0/3410002780' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:58:42.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:58:42 smithi088 bash[61791]: ts=2024-09-13T00:58:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:58:42.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:42 smithi077 bash[80681]: cluster 2024-09-13T00:58:41.015544+0000 mgr.y (mgr.25304) 686 : cluster [DBG] pgmap v1260: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:42.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:42 smithi077 bash[80681]: audit 2024-09-13T00:58:41.630915+0000 mon.a (mon.0) 1589 : audit [DBG] from='client.? 172.21.15.77:0/3410002780' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:58:42.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:42 smithi077 bash[83127]: cluster 2024-09-13T00:58:41.015544+0000 mgr.y (mgr.25304) 686 : cluster [DBG] pgmap v1260: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:42.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:42 smithi077 bash[83127]: audit 2024-09-13T00:58:41.630915+0000 mon.a (mon.0) 1589 : audit [DBG] from='client.? 172.21.15.77:0/3410002780' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:58:43.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:43 smithi077 bash[80681]: audit 2024-09-13T00:58:42.896408+0000 mon.a (mon.0) 1590 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:58:43.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:43 smithi077 bash[80681]: audit 2024-09-13T00:58:43.438899+0000 mon.a (mon.0) 1591 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:58:43.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:43 smithi077 bash[83127]: audit 2024-09-13T00:58:42.896408+0000 mon.a (mon.0) 1590 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:58:43.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:43 smithi077 bash[83127]: audit 2024-09-13T00:58:43.438899+0000 mon.a (mon.0) 1591 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:58:43.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:43 smithi088 bash[68960]: audit 2024-09-13T00:58:42.896408+0000 mon.a (mon.0) 1590 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-13T00:58:43.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:43 smithi088 bash[68960]: audit 2024-09-13T00:58:43.438899+0000 mon.a (mon.0) 1591 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:58:44.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:44 smithi077 bash[80681]: cluster 2024-09-13T00:58:43.016348+0000 mgr.y (mgr.25304) 687 : cluster [DBG] pgmap v1261: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:44.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:44 smithi077 bash[83127]: cluster 2024-09-13T00:58:43.016348+0000 mgr.y (mgr.25304) 687 : cluster [DBG] pgmap v1261: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:44.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:44 smithi088 bash[68960]: cluster 2024-09-13T00:58:43.016348+0000 mgr.y (mgr.25304) 687 : cluster [DBG] pgmap v1261: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:45.396 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:58:46.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:46 smithi088 bash[68960]: cluster 2024-09-13T00:58:45.017362+0000 mgr.y (mgr.25304) 688 : cluster [DBG] pgmap v1262: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:47.028 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:46 smithi077 bash[80681]: cluster 2024-09-13T00:58:45.017362+0000 mgr.y (mgr.25304) 688 : cluster [DBG] pgmap v1262: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:47.029 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:46 smithi077 bash[83127]: cluster 2024-09-13T00:58:45.017362+0000 mgr.y (mgr.25304) 688 : cluster [DBG] pgmap v1262: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:48.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:58:48 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:58:48] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-09-13T00:58:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:48 smithi077 bash[80681]: cluster 2024-09-13T00:58:47.018053+0000 mgr.y (mgr.25304) 689 : cluster [DBG] pgmap v1263: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:48 smithi077 bash[80681]: audit 2024-09-13T00:58:48.482574+0000 mon.a (mon.0) 1592 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:58:48.779 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:48 smithi077 bash[80681]: audit 2024-09-13T00:58:48.490837+0000 mon.a (mon.0) 1593 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:58:48.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:48 smithi077 bash[83127]: cluster 2024-09-13T00:58:47.018053+0000 mgr.y (mgr.25304) 689 : cluster [DBG] pgmap v1263: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:48.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:48 smithi077 bash[83127]: audit 2024-09-13T00:58:48.482574+0000 mon.a (mon.0) 1592 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:58:48.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:48 smithi077 bash[83127]: audit 2024-09-13T00:58:48.490837+0000 mon.a (mon.0) 1593 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:58:48.852 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:48 smithi088 bash[68960]: cluster 2024-09-13T00:58:47.018053+0000 mgr.y (mgr.25304) 689 : cluster [DBG] pgmap v1263: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:48 smithi088 bash[68960]: audit 2024-09-13T00:58:48.482574+0000 mon.a (mon.0) 1592 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:58:48.853 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:48 smithi088 bash[68960]: audit 2024-09-13T00:58:48.490837+0000 mon.a (mon.0) 1593 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:58:49.847 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:58:50.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:49 smithi077 bash[80681]: audit 2024-09-13T00:58:48.744760+0000 mon.a (mon.0) 1594 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:58:50.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:49 smithi077 bash[80681]: audit 2024-09-13T00:58:48.751022+0000 mon.a (mon.0) 1595 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:58:50.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:49 smithi077 bash[80681]: cluster 2024-09-13T00:58:49.018927+0000 mgr.y (mgr.25304) 690 : cluster [DBG] pgmap v1264: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:50.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:49 smithi077 bash[80681]: audit 2024-09-13T00:58:49.151050+0000 mon.a (mon.0) 1596 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:58:50.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:49 smithi077 bash[80681]: audit 2024-09-13T00:58:49.152226+0000 mon.a (mon.0) 1597 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:58:50.030 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:49 smithi077 bash[80681]: audit 2024-09-13T00:58:49.160694+0000 mon.a (mon.0) 1598 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:58:50.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:49 smithi077 bash[83127]: audit 2024-09-13T00:58:48.744760+0000 mon.a (mon.0) 1594 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:58:50.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:49 smithi077 bash[83127]: audit 2024-09-13T00:58:48.751022+0000 mon.a (mon.0) 1595 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:58:50.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:49 smithi077 bash[83127]: cluster 2024-09-13T00:58:49.018927+0000 mgr.y (mgr.25304) 690 : cluster [DBG] pgmap v1264: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:50.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:49 smithi077 bash[83127]: audit 2024-09-13T00:58:49.151050+0000 mon.a (mon.0) 1596 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:58:50.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:49 smithi077 bash[83127]: audit 2024-09-13T00:58:49.152226+0000 mon.a (mon.0) 1597 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:58:50.030 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:49 smithi077 bash[83127]: audit 2024-09-13T00:58:49.160694+0000 mon.a (mon.0) 1598 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:58:50.102 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:49 smithi088 bash[68960]: audit 2024-09-13T00:58:48.744760+0000 mon.a (mon.0) 1594 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:58:50.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:49 smithi088 bash[68960]: audit 2024-09-13T00:58:48.751022+0000 mon.a (mon.0) 1595 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:58:50.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:49 smithi088 bash[68960]: cluster 2024-09-13T00:58:49.018927+0000 mgr.y (mgr.25304) 690 : cluster [DBG] pgmap v1264: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:50.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:49 smithi088 bash[68960]: audit 2024-09-13T00:58:49.151050+0000 mon.a (mon.0) 1596 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-13T00:58:50.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:49 smithi088 bash[68960]: audit 2024-09-13T00:58:49.152226+0000 mon.a (mon.0) 1597 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-13T00:58:50.103 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:49 smithi088 bash[68960]: audit 2024-09-13T00:58:49.160694+0000 mon.a (mon.0) 1598 : audit [INF] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' 2024-09-13T00:58:52.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:52 smithi088 bash[68960]: cluster 2024-09-13T00:58:51.019779+0000 mgr.y (mgr.25304) 691 : cluster [DBG] pgmap v1265: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:52.352 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:58:52 smithi088 bash[61791]: ts=2024-09-13T00:58:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:58:52.530 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:52 smithi077 bash[83127]: cluster 2024-09-13T00:58:51.019779+0000 mgr.y (mgr.25304) 691 : cluster [DBG] pgmap v1265: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:52.530 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:52 smithi077 bash[80681]: cluster 2024-09-13T00:58:51.019779+0000 mgr.y (mgr.25304) 691 : cluster [DBG] pgmap v1265: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:53.691 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:58:53.691 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:58:54.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:54 smithi088 bash[68960]: cluster 2024-09-13T00:58:53.020339+0000 mgr.y (mgr.25304) 692 : cluster [DBG] pgmap v1266: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:54.353 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:54 smithi088 bash[68960]: audit 2024-09-13T00:58:53.693894+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.77:0/1942100566' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:58:54.438 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:54 smithi077 bash[80681]: cluster 2024-09-13T00:58:53.020339+0000 mgr.y (mgr.25304) 692 : cluster [DBG] pgmap v1266: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:54.438 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:54 smithi077 bash[80681]: audit 2024-09-13T00:58:53.693894+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.77:0/1942100566' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:58:54.438 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:54 smithi077 bash[83127]: cluster 2024-09-13T00:58:53.020339+0000 mgr.y (mgr.25304) 692 : cluster [DBG] pgmap v1266: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:54.438 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:54 smithi077 bash[83127]: audit 2024-09-13T00:58:53.693894+0000 mon.c (mon.2) 60 : audit [DBG] from='client.? 172.21.15.77:0/1942100566' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:58:56.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:56 smithi088 bash[68960]: cluster 2024-09-13T00:58:55.021246+0000 mgr.y (mgr.25304) 693 : cluster [DBG] pgmap v1267: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:56.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:56 smithi077 bash[83127]: cluster 2024-09-13T00:58:55.021246+0000 mgr.y (mgr.25304) 693 : cluster [DBG] pgmap v1267: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:56.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:56 smithi077 bash[80681]: cluster 2024-09-13T00:58:55.021246+0000 mgr.y (mgr.25304) 693 : cluster [DBG] pgmap v1267: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:57.440 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:58:58.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:58 smithi088 bash[68960]: cluster 2024-09-13T00:58:57.022024+0000 mgr.y (mgr.25304) 694 : cluster [DBG] pgmap v1268: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:58.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:58 smithi077 bash[80681]: cluster 2024-09-13T00:58:57.022024+0000 mgr.y (mgr.25304) 694 : cluster [DBG] pgmap v1268: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:58.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:58 smithi077 bash[83127]: cluster 2024-09-13T00:58:57.022024+0000 mgr.y (mgr.25304) 694 : cluster [DBG] pgmap v1268: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:58:58.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:58:58 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:58:58] "GET /metrics HTTP/1.1" 200 34261 "" "Prometheus/2.43.0" 2024-09-13T00:58:59.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:58:59 smithi088 bash[68960]: audit 2024-09-13T00:58:58.439359+0000 mon.a (mon.0) 1599 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:58:59.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:58:59 smithi077 bash[80681]: audit 2024-09-13T00:58:58.439359+0000 mon.a (mon.0) 1599 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:58:59.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:58:59 smithi077 bash[83127]: audit 2024-09-13T00:58:58.439359+0000 mon.a (mon.0) 1599 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:59:00.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:00 smithi088 bash[68960]: cluster 2024-09-13T00:58:59.022824+0000 mgr.y (mgr.25304) 695 : cluster [DBG] pgmap v1269: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:00.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:00 smithi077 bash[80681]: cluster 2024-09-13T00:58:59.022824+0000 mgr.y (mgr.25304) 695 : cluster [DBG] pgmap v1269: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:00.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:00 smithi077 bash[83127]: cluster 2024-09-13T00:58:59.022824+0000 mgr.y (mgr.25304) 695 : cluster [DBG] pgmap v1269: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:02.229 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:59:02.352 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:02 smithi088 bash[68960]: cluster 2024-09-13T00:59:01.023723+0000 mgr.y (mgr.25304) 696 : cluster [DBG] pgmap v1270: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:02.353 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:59:02 smithi088 bash[61791]: ts=2024-09-13T00:59:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:59:02.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:02 smithi077 bash[80681]: cluster 2024-09-13T00:59:01.023723+0000 mgr.y (mgr.25304) 696 : cluster [DBG] pgmap v1270: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:02.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:02 smithi077 bash[83127]: cluster 2024-09-13T00:59:01.023723+0000 mgr.y (mgr.25304) 696 : cluster [DBG] pgmap v1270: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:04.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:04 smithi077 bash[80681]: cluster 2024-09-13T00:59:03.024631+0000 mgr.y (mgr.25304) 697 : cluster [DBG] pgmap v1271: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:04.279 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:04 smithi077 bash[83127]: cluster 2024-09-13T00:59:03.024631+0000 mgr.y (mgr.25304) 697 : cluster [DBG] pgmap v1271: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:04.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:04 smithi088 bash[68960]: cluster 2024-09-13T00:59:03.024631+0000 mgr.y (mgr.25304) 697 : cluster [DBG] pgmap v1271: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:06.102 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:59:06.102 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:59:06.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:06 smithi077 bash[80681]: cluster 2024-09-13T00:59:05.025747+0000 mgr.y (mgr.25304) 698 : cluster [DBG] pgmap v1272: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:06.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:06 smithi077 bash[80681]: audit 2024-09-13T00:59:06.104261+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.77:0/127564105' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:59:06.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:06 smithi077 bash[83127]: cluster 2024-09-13T00:59:05.025747+0000 mgr.y (mgr.25304) 698 : cluster [DBG] pgmap v1272: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:06.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:06 smithi077 bash[83127]: audit 2024-09-13T00:59:06.104261+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.77:0/127564105' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:59:06.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:06 smithi088 bash[68960]: cluster 2024-09-13T00:59:05.025747+0000 mgr.y (mgr.25304) 698 : cluster [DBG] pgmap v1272: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:06.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:06 smithi088 bash[68960]: audit 2024-09-13T00:59:06.104261+0000 mon.c (mon.2) 61 : audit [DBG] from='client.? 172.21.15.77:0/127564105' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:59:08.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:08 smithi077 bash[83127]: cluster 2024-09-13T00:59:07.026550+0000 mgr.y (mgr.25304) 699 : cluster [DBG] pgmap v1273: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:08.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:08 smithi077 bash[80681]: cluster 2024-09-13T00:59:07.026550+0000 mgr.y (mgr.25304) 699 : cluster [DBG] pgmap v1273: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:08.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:08 smithi088 bash[68960]: cluster 2024-09-13T00:59:07.026550+0000 mgr.y (mgr.25304) 699 : cluster [DBG] pgmap v1273: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:08.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:08 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:59:08] "GET /metrics HTTP/1.1" 200 34261 "" "Prometheus/2.43.0" 2024-09-13T00:59:09.911 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:59:10.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:10 smithi077 bash[80681]: cluster 2024-09-13T00:59:09.027367+0000 mgr.y (mgr.25304) 700 : cluster [DBG] pgmap v1274: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:10.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:10 smithi077 bash[83127]: cluster 2024-09-13T00:59:09.027367+0000 mgr.y (mgr.25304) 700 : cluster [DBG] pgmap v1274: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:10.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:10 smithi088 bash[68960]: cluster 2024-09-13T00:59:09.027367+0000 mgr.y (mgr.25304) 700 : cluster [DBG] pgmap v1274: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:12.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:12 smithi077 bash[80681]: cluster 2024-09-13T00:59:11.028275+0000 mgr.y (mgr.25304) 701 : cluster [DBG] pgmap v1275: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:12.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:12 smithi077 bash[83127]: cluster 2024-09-13T00:59:11.028275+0000 mgr.y (mgr.25304) 701 : cluster [DBG] pgmap v1275: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:12.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:12 smithi088 bash[68960]: cluster 2024-09-13T00:59:11.028275+0000 mgr.y (mgr.25304) 701 : cluster [DBG] pgmap v1275: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:12.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:59:12 smithi088 bash[61791]: ts=2024-09-13T00:59:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:59:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:14 smithi077 bash[80681]: cluster 2024-09-13T00:59:13.029130+0000 mgr.y (mgr.25304) 702 : cluster [DBG] pgmap v1276: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:14.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:14 smithi077 bash[80681]: audit 2024-09-13T00:59:13.440009+0000 mon.a (mon.0) 1600 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:59:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:14 smithi077 bash[83127]: cluster 2024-09-13T00:59:13.029130+0000 mgr.y (mgr.25304) 702 : cluster [DBG] pgmap v1276: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:14.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:14 smithi077 bash[83127]: audit 2024-09-13T00:59:13.440009+0000 mon.a (mon.0) 1600 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:59:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:14 smithi088 bash[68960]: cluster 2024-09-13T00:59:13.029130+0000 mgr.y (mgr.25304) 702 : cluster [DBG] pgmap v1276: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:14.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:14 smithi088 bash[68960]: audit 2024-09-13T00:59:13.440009+0000 mon.a (mon.0) 1600 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:59:14.699 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:59:16.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:16 smithi077 bash[80681]: cluster 2024-09-13T00:59:15.030103+0000 mgr.y (mgr.25304) 703 : cluster [DBG] pgmap v1277: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:16.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:16 smithi077 bash[83127]: cluster 2024-09-13T00:59:15.030103+0000 mgr.y (mgr.25304) 703 : cluster [DBG] pgmap v1277: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:16.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:16 smithi088 bash[68960]: cluster 2024-09-13T00:59:15.030103+0000 mgr.y (mgr.25304) 703 : cluster [DBG] pgmap v1277: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:18.278 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:18 smithi077 bash[83127]: cluster 2024-09-13T00:59:17.030681+0000 mgr.y (mgr.25304) 704 : cluster [DBG] pgmap v1278: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:18.279 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:18 smithi077 bash[80681]: cluster 2024-09-13T00:59:17.030681+0000 mgr.y (mgr.25304) 704 : cluster [DBG] pgmap v1278: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:18.394 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:59:18.394 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:59:18.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:18 smithi088 bash[68960]: cluster 2024-09-13T00:59:17.030681+0000 mgr.y (mgr.25304) 704 : cluster [DBG] pgmap v1278: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:18.778 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:18 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:59:18] "GET /metrics HTTP/1.1" 200 34263 "" "Prometheus/2.43.0" 2024-09-13T00:59:19.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:19 smithi077 bash[80681]: audit 2024-09-13T00:59:18.396621+0000 mon.a (mon.0) 1601 : audit [DBG] from='client.? 172.21.15.77:0/2351096307' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:59:19.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:19 smithi077 bash[83127]: audit 2024-09-13T00:59:18.396621+0000 mon.a (mon.0) 1601 : audit [DBG] from='client.? 172.21.15.77:0/2351096307' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:59:19.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:19 smithi088 bash[68960]: audit 2024-09-13T00:59:18.396621+0000 mon.a (mon.0) 1601 : audit [DBG] from='client.? 172.21.15.77:0/2351096307' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:59:20.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:20 smithi077 bash[80681]: cluster 2024-09-13T00:59:19.031551+0000 mgr.y (mgr.25304) 705 : cluster [DBG] pgmap v1279: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:20.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:20 smithi077 bash[83127]: cluster 2024-09-13T00:59:19.031551+0000 mgr.y (mgr.25304) 705 : cluster [DBG] pgmap v1279: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:20.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:20 smithi088 bash[68960]: cluster 2024-09-13T00:59:19.031551+0000 mgr.y (mgr.25304) 705 : cluster [DBG] pgmap v1279: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:22.205 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 9218306c-7165-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-13T00:59:22.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:22 smithi077 bash[80681]: cluster 2024-09-13T00:59:21.032557+0000 mgr.y (mgr.25304) 706 : cluster [DBG] pgmap v1280: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:22.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:22 smithi077 bash[83127]: cluster 2024-09-13T00:59:21.032557+0000 mgr.y (mgr.25304) 706 : cluster [DBG] pgmap v1280: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:22.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:22 smithi088 bash[68960]: cluster 2024-09-13T00:59:21.032557+0000 mgr.y (mgr.25304) 706 : cluster [DBG] pgmap v1280: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:22.603 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:59:22 smithi088 bash[61791]: ts=2024-09-13T00:59:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:59:24.528 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:24 smithi077 bash[80681]: cluster 2024-09-13T00:59:23.033487+0000 mgr.y (mgr.25304) 707 : cluster [DBG] pgmap v1281: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:24.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:24 smithi077 bash[83127]: cluster 2024-09-13T00:59:23.033487+0000 mgr.y (mgr.25304) 707 : cluster [DBG] pgmap v1281: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:24.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:24 smithi088 bash[68960]: cluster 2024-09-13T00:59:23.033487+0000 mgr.y (mgr.25304) 707 : cluster [DBG] pgmap v1281: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:26.603 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:26 smithi088 bash[68960]: cluster 2024-09-13T00:59:25.034212+0000 mgr.y (mgr.25304) 708 : cluster [DBG] pgmap v1282: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:26.778 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:26 smithi077 bash[80681]: cluster 2024-09-13T00:59:25.034212+0000 mgr.y (mgr.25304) 708 : cluster [DBG] pgmap v1282: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:26.779 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:26 smithi077 bash[83127]: cluster 2024-09-13T00:59:25.034212+0000 mgr.y (mgr.25304) 708 : cluster [DBG] pgmap v1282: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:26.996 INFO:teuthology.orchestra.run.smithi077.stderr:Inferring config /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/mon.a/config 2024-09-13T00:59:28.527 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:28 smithi077 bash[80681]: cluster 2024-09-13T00:59:27.034868+0000 mgr.y (mgr.25304) 709 : cluster [DBG] pgmap v1283: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:28.528 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:28 smithi077 bash[83127]: cluster 2024-09-13T00:59:27.034868+0000 mgr.y (mgr.25304) 709 : cluster [DBG] pgmap v1283: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:28.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:28 smithi088 bash[68960]: cluster 2024-09-13T00:59:27.034868+0000 mgr.y (mgr.25304) 709 : cluster [DBG] pgmap v1283: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:28.779 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:28 smithi077 bash[73137]: ::ffff:172.21.15.88 - - [13/Sep/2024:00:59:28] "GET /metrics HTTP/1.1" 200 34262 "" "Prometheus/2.43.0" 2024-09-13T00:59:29.529 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:29 smithi077 bash[80681]: audit 2024-09-13T00:59:28.439929+0000 mon.a (mon.0) 1602 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:59:29.529 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:29 smithi077 bash[83127]: audit 2024-09-13T00:59:28.439929+0000 mon.a (mon.0) 1602 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:59:29.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:29 smithi088 bash[68960]: audit 2024-09-13T00:59:28.439929+0000 mon.a (mon.0) 1602 : audit [DBG] from='mgr.25304 172.21.15.77:0/1065098036' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-13T00:59:30.602 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:30 smithi088 bash[68960]: cluster 2024-09-13T00:59:29.035555+0000 mgr.y (mgr.25304) 710 : cluster [DBG] pgmap v1284: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:30.643 INFO:teuthology.orchestra.run.smithi077.stdout: 2024-09-13T00:59:30.644 INFO:teuthology.orchestra.run.smithi077.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-09-13T00:59:30.655 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:30 smithi077 bash[80681]: cluster 2024-09-13T00:59:29.035555+0000 mgr.y (mgr.25304) 710 : cluster [DBG] pgmap v1284: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:30.655 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:30 smithi077 bash[83127]: cluster 2024-09-13T00:59:29.035555+0000 mgr.y (mgr.25304) 710 : cluster [DBG] pgmap v1284: 73 pgs: 73 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-13T00:59:31.466 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_7a853b1706d076d58e0da7a5d3b9b9eb36276880/qa/tasks/ceph.py", line 1489, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_7a853b1706d076d58e0da7a5d3b9b9eb36276880/qa/tasks/ceph_manager.py", line 3201, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-09-13T00:59:31.725 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=718e0974f06247e2a1fc22a5b88abc7c Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_7a853b1706d076d58e0da7a5d3b9b9eb36276880/qa/tasks/ceph.py", line 1489, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_7a853b1706d076d58e0da7a5d3b9b9eb36276880/qa/tasks/ceph_manager.py", line 3201, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-09-13T00:59:31.727 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-13T00:59:31.737 INFO:tasks.cephadm:Teardown begin 2024-09-13T00:59:31.738 DEBUG:teuthology.orchestra.run.smithi077:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T00:59:31.742 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:31 smithi088 bash[68960]: audit 2024-09-13T00:59:30.646260+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.77:0/2472447125' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:59:31.752 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:31 smithi077 bash[80681]: audit 2024-09-13T00:59:30.646260+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.77:0/2472447125' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:59:31.752 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:31 smithi077 bash[83127]: audit 2024-09-13T00:59:30.646260+0000 mon.c (mon.2) 62 : audit [DBG] from='client.? 172.21.15.77:0/2472447125' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-13T00:59:31.756 DEBUG:teuthology.orchestra.run.smithi088:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T00:59:31.771 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-13T00:59:31.771 DEBUG:teuthology.orchestra.run.smithi077:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T00:59:31.804 DEBUG:teuthology.orchestra.run.smithi088:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-13T00:59:31.820 INFO:tasks.cephadm:Stopping all daemons... 2024-09-13T00:59:31.820 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-13T00:59:31.821 DEBUG:teuthology.orchestra.run.smithi077:> sudo systemctl stop ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.a 2024-09-13T00:59:32.028 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:31 smithi077 bash[73137]: [13/Sep/2024:00:59:31] ENGINE Bus STOPPING 2024-09-13T00:59:32.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:31 smithi077 systemd[1]: Stopping Ceph mon.a for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:59:32.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:31 smithi077 bash[80681]: debug 2024-09-13T00:59:31.978+0000 7fc79f5c6640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T00:59:32.029 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:31 smithi077 bash[80681]: debug 2024-09-13T00:59:31.978+0000 7fc79f5c6640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-13T00:59:32.135 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:32 smithi077 bash[73137]: [13/Sep/2024:00:59:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:59:32.135 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:32 smithi077 bash[73137]: [13/Sep/2024:00:59:32] ENGINE Bus STOPPED 2024-09-13T00:59:32.135 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:32 smithi077 bash[73137]: [13/Sep/2024:00:59:32] ENGINE Bus STARTING 2024-09-13T00:59:32.135 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:32 smithi077 bash[73137]: [13/Sep/2024:00:59:32] ENGINE Serving on http://:::9283 2024-09-13T00:59:32.442 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:32 smithi077 bash[73137]: [13/Sep/2024:00:59:32] ENGINE Bus STARTED 2024-09-13T00:59:32.602 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 00:59:32 smithi088 bash[61791]: ts=2024-09-13T00:59:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"ceph_cluster\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"9218306c-7165-11ef-bceb-c7b262605968\", instance=\"ceph_cluster\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-09-13T00:59:33.093 DEBUG:teuthology.orchestra.run.smithi077:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.a.service' 2024-09-13T00:59:33.103 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:32 smithi077 bash[123235]: ceph-9218306c-7165-11ef-bceb-c7b262605968-mon-a 2024-09-13T00:59:33.104 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:33 smithi077 bash[123306]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-mon-a 2024-09-13T00:59:33.104 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:33 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-13T00:59:33.104 INFO:journalctl@ceph.mon.a.smithi077.stdout:Sep 13 00:59:33 smithi077 systemd[1]: Stopped Ceph mon.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T00:59:33.125 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:59:33.125 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-13T00:59:33.125 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-13T00:59:33.125 DEBUG:teuthology.orchestra.run.smithi077:> sudo systemctl stop ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.c 2024-09-13T00:59:33.354 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:33 smithi077 bash[73137]: [13/Sep/2024:00:59:33] ENGINE Bus STOPPING 2024-09-13T00:59:33.355 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:33 smithi077 systemd[1]: Stopping Ceph mon.c for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:59:33.355 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:33 smithi077 bash[83127]: debug 2024-09-13T00:59:33.306+0000 7f997f8ea640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T00:59:33.355 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:33 smithi077 bash[83127]: debug 2024-09-13T00:59:33.306+0000 7f997f8ea640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-09-13T00:59:33.989 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:33 smithi077 bash[73137]: [13/Sep/2024:00:59:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-13T00:59:33.989 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:33 smithi077 bash[73137]: [13/Sep/2024:00:59:33] ENGINE Bus STOPPED 2024-09-13T00:59:33.989 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:33 smithi077 bash[73137]: [13/Sep/2024:00:59:33] ENGINE Bus STARTING 2024-09-13T00:59:33.989 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:33 smithi077 bash[73137]: [13/Sep/2024:00:59:33] ENGINE Serving on http://:::9283 2024-09-13T00:59:33.989 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:33 smithi077 bash[73137]: [13/Sep/2024:00:59:33] ENGINE Bus STARTED 2024-09-13T00:59:34.240 INFO:journalctl@ceph.mon.c.smithi077.stdout:Sep 13 00:59:33 smithi077 bash[123348]: ceph-9218306c-7165-11ef-bceb-c7b262605968-mon-c 2024-09-13T00:59:34.255 DEBUG:teuthology.orchestra.run.smithi077:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.c.service' 2024-09-13T00:59:34.279 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:59:34.279 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-13T00:59:34.280 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-13T00:59:34.280 DEBUG:teuthology.orchestra.run.smithi088:> sudo systemctl stop ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.b 2024-09-13T00:59:34.543 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:34 smithi088 systemd[1]: Stopping Ceph mon.b for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:59:34.543 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:34 smithi088 bash[68960]: debug 2024-09-13T00:59:34.478+0000 7f4ed651b640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-09-13T00:59:34.543 INFO:journalctl@ceph.mon.b.smithi088.stdout:Sep 13 00:59:34 smithi088 bash[68960]: debug 2024-09-13T00:59:34.478+0000 7f4ed651b640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-09-13T00:59:35.299 DEBUG:teuthology.orchestra.run.smithi088:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@mon.b.service' 2024-09-13T00:59:35.332 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:59:35.333 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-13T00:59:35.333 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-13T00:59:35.333 DEBUG:teuthology.orchestra.run.smithi077:> sudo systemctl stop ceph-9218306c-7165-11ef-bceb-c7b262605968@mgr.y 2024-09-13T00:59:35.601 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:35 smithi077 systemd[1]: Stopping Ceph mgr.y for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:59:35.601 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:35 smithi077 bash[73137]: debug 2024-09-13T00:59:35.506+0000 7ffaf16d6640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:59:35.602 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:35 smithi077 bash[73137]: debug 2024-09-13T00:59:35.506+0000 7ffaf16d6640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-09-13T00:59:36.426 INFO:journalctl@ceph.mgr.y.smithi077.stdout:Sep 13 00:59:36 smithi077 bash[123458]: ceph-9218306c-7165-11ef-bceb-c7b262605968-mgr-y 2024-09-13T00:59:36.441 DEBUG:teuthology.orchestra.run.smithi077:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@mgr.y.service' 2024-09-13T00:59:36.463 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:59:36.464 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-13T00:59:36.464 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-13T00:59:36.464 DEBUG:teuthology.orchestra.run.smithi088:> sudo systemctl stop ceph-9218306c-7165-11ef-bceb-c7b262605968@mgr.x 2024-09-13T00:59:36.835 INFO:journalctl@ceph.mgr.x.smithi088.stdout:Sep 13 00:59:36 smithi088 systemd[1]: Stopping Ceph mgr.x for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:59:37.317 DEBUG:teuthology.orchestra.run.smithi088:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@mgr.x.service' 2024-09-13T00:59:37.339 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:59:37.340 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-13T00:59:37.340 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-13T00:59:37.340 DEBUG:teuthology.orchestra.run.smithi077:> sudo systemctl stop ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.0 2024-09-13T00:59:37.779 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:59:37 smithi077 systemd[1]: Stopping Ceph osd.0 for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:59:37.780 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:59:37 smithi077 bash[89778]: debug 2024-09-13T00:59:37.469+0000 7fd06db49640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:59:37.781 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:59:37 smithi077 bash[89778]: debug 2024-09-13T00:59:37.469+0000 7fd06db49640 -1 osd.0 956 *** Got signal Terminated *** 2024-09-13T00:59:37.782 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:59:37 smithi077 bash[89778]: debug 2024-09-13T00:59:37.469+0000 7fd06db49640 -1 osd.0 956 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T00:59:40.528 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:59:40 smithi077 bash[72541]: ts=2024-09-13T00:59:40.204Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused" 2024-09-13T00:59:40.529 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:59:40 smithi077 bash[72541]: ts=2024-09-13T00:59:40.206Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused" 2024-09-13T00:59:43.778 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:59:43 smithi077 bash[123556]: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-0 2024-09-13T00:59:43.779 INFO:journalctl@ceph.osd.0.smithi077.stdout:Sep 13 00:59:43 smithi077 bash[123620]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-0 2024-09-13T00:59:45.819 DEBUG:teuthology.orchestra.run.smithi077:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.0.service' 2024-09-13T00:59:45.846 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:59:45.846 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-13T00:59:45.846 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-13T00:59:45.846 DEBUG:teuthology.orchestra.run.smithi077:> sudo systemctl stop ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.1 2024-09-13T00:59:46.278 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:59:45 smithi077 systemd[1]: Stopping Ceph osd.1 for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:59:46.279 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:59:46 smithi077 bash[94733]: debug 2024-09-13T00:59:46.029+0000 7f1349fc6640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:59:46.279 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:59:46 smithi077 bash[94733]: debug 2024-09-13T00:59:46.029+0000 7f1349fc6640 -1 osd.1 956 *** Got signal Terminated *** 2024-09-13T00:59:46.279 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:59:46 smithi077 bash[94733]: debug 2024-09-13T00:59:46.029+0000 7f1349fc6640 -1 osd.1 956 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T00:59:50.528 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:59:50 smithi077 bash[72541]: ts=2024-09-13T00:59:50.204Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused" 2024-09-13T00:59:50.529 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:59:50 smithi077 bash[72541]: ts=2024-09-13T00:59:50.204Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused" 2024-09-13T00:59:50.529 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 00:59:50 smithi077 bash[72541]: ts=2024-09-13T00:59:50.206Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused" 2024-09-13T00:59:52.340 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:59:52 smithi077 bash[123754]: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-1 2024-09-13T00:59:52.778 INFO:journalctl@ceph.osd.1.smithi077.stdout:Sep 13 00:59:52 smithi077 bash[123822]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-1 2024-09-13T00:59:54.243 DEBUG:teuthology.orchestra.run.smithi077:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.1.service' 2024-09-13T00:59:54.272 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T00:59:54.273 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-13T00:59:54.273 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-13T00:59:54.273 DEBUG:teuthology.orchestra.run.smithi077:> sudo systemctl stop ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.2 2024-09-13T00:59:54.528 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:59:54 smithi077 systemd[1]: Stopping Ceph osd.2 for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T00:59:54.529 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:59:54 smithi077 bash[100067]: debug 2024-09-13T00:59:54.457+0000 7fadba73f640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T00:59:54.529 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:59:54 smithi077 bash[100067]: debug 2024-09-13T00:59:54.457+0000 7fadba73f640 -1 osd.2 956 *** Got signal Terminated *** 2024-09-13T00:59:54.529 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 00:59:54 smithi077 bash[100067]: debug 2024-09-13T00:59:54.457+0000 7fadba73f640 -1 osd.2 956 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T01:00:00.528 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 01:00:00 smithi077 bash[123952]: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-2 2024-09-13T01:00:00.529 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:00 smithi077 bash[72541]: ts=2024-09-13T01:00:00.205Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused" 2024-09-13T01:00:00.529 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:00 smithi077 bash[72541]: ts=2024-09-13T01:00:00.205Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused" 2024-09-13T01:00:00.529 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:00 smithi077 bash[72541]: ts=2024-09-13T01:00:00.207Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused" 2024-09-13T01:00:01.028 INFO:journalctl@ceph.osd.2.smithi077.stdout:Sep 13 01:00:00 smithi077 bash[124017]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-2 2024-09-13T01:00:02.769 DEBUG:teuthology.orchestra.run.smithi077:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.2.service' 2024-09-13T01:00:02.797 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T01:00:02.797 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-13T01:00:02.798 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-13T01:00:02.798 DEBUG:teuthology.orchestra.run.smithi077:> sudo systemctl stop ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.3 2024-09-13T01:00:03.028 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 01:00:02 smithi077 systemd[1]: Stopping Ceph osd.3 for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T01:00:03.029 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 01:00:02 smithi077 bash[104829]: debug 2024-09-13T01:00:02.985+0000 7f86319d5640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T01:00:03.029 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 01:00:02 smithi077 bash[104829]: debug 2024-09-13T01:00:02.985+0000 7f86319d5640 -1 osd.3 956 *** Got signal Terminated *** 2024-09-13T01:00:03.029 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 01:00:02 smithi077 bash[104829]: debug 2024-09-13T01:00:02.985+0000 7f86319d5640 -1 osd.3 956 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T01:00:03.852 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:03 smithi088 bash[74473]: debug 2024-09-13T01:00:03.494+0000 7f01765f1640 -1 osd.4 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:40.186921+0000 front 2024-09-13T00:59:40.186555+0000 (oldest deadline 2024-09-13T01:00:03.086479+0000) 2024-09-13T01:00:04.603 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:04 smithi088 bash[74473]: debug 2024-09-13T01:00:04.450+0000 7f01765f1640 -1 osd.4 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:40.186921+0000 front 2024-09-13T00:59:40.186555+0000 (oldest deadline 2024-09-13T01:00:03.086479+0000) 2024-09-13T01:00:04.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:04 smithi088 bash[45092]: debug 2024-09-13T01:00:04.218+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:05.103 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:04 smithi088 bash[33847]: debug 2024-09-13T01:00:04.726+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:05.468 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:05 smithi088 bash[45092]: debug 2024-09-13T01:00:05.202+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:05.528 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 01:00:05 smithi077 bash[104829]: debug 2024-09-13T01:00:05.217+0000 7f8629fe1640 -1 osd.3 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:40.638780+0000 front 2024-09-13T00:59:40.639058+0000 (oldest deadline 2024-09-13T01:00:04.738931+0000) 2024-09-13T01:00:05.853 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:05 smithi088 bash[33847]: debug 2024-09-13T01:00:05.698+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:05.853 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:05 smithi088 bash[74473]: debug 2024-09-13T01:00:05.466+0000 7f01765f1640 -1 osd.4 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:40.186921+0000 front 2024-09-13T00:59:40.186555+0000 (oldest deadline 2024-09-13T01:00:03.086479+0000) 2024-09-13T01:00:06.492 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:06 smithi088 bash[45092]: debug 2024-09-13T01:00:06.209+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:06.528 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 01:00:06 smithi077 bash[104829]: debug 2024-09-13T01:00:06.241+0000 7f8629fe1640 -1 osd.3 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:40.638780+0000 front 2024-09-13T00:59:40.639058+0000 (oldest deadline 2024-09-13T01:00:04.738931+0000) 2024-09-13T01:00:06.853 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:06 smithi088 bash[74473]: debug 2024-09-13T01:00:06.489+0000 7f01765f1640 -1 osd.4 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:40.186921+0000 front 2024-09-13T00:59:40.186555+0000 (oldest deadline 2024-09-13T01:00:03.086479+0000) 2024-09-13T01:00:06.853 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:06 smithi088 bash[33847]: debug 2024-09-13T01:00:06.725+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:07.255 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:06 smithi088 bash[39322]: debug 2024-09-13T01:00:06.921+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:07.528 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 01:00:07 smithi077 bash[104829]: debug 2024-09-13T01:00:07.261+0000 7f8629fe1640 -1 osd.3 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:40.638780+0000 front 2024-09-13T00:59:40.639058+0000 (oldest deadline 2024-09-13T01:00:04.738931+0000) 2024-09-13T01:00:07.531 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:07 smithi088 bash[45092]: debug 2024-09-13T01:00:07.253+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:07.852 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:07 smithi088 bash[33847]: debug 2024-09-13T01:00:07.741+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:07.853 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:07 smithi088 bash[74473]: debug 2024-09-13T01:00:07.529+0000 7f01765f1640 -1 osd.4 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:40.186921+0000 front 2024-09-13T00:59:40.186555+0000 (oldest deadline 2024-09-13T01:00:03.086479+0000) 2024-09-13T01:00:08.282 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:07 smithi088 bash[39322]: debug 2024-09-13T01:00:07.913+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:08.603 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:08 smithi088 bash[74473]: debug 2024-09-13T01:00:08.517+0000 7f01765f1640 -1 osd.4 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:40.186921+0000 front 2024-09-13T00:59:40.186555+0000 (oldest deadline 2024-09-13T01:00:03.086479+0000) 2024-09-13T01:00:08.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:08 smithi088 bash[45092]: debug 2024-09-13T01:00:08.277+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:09.103 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:08 smithi088 bash[33847]: debug 2024-09-13T01:00:08.793+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:09.103 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:08 smithi088 bash[39322]: debug 2024-09-13T01:00:08.937+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:09.203 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 01:00:08 smithi077 bash[124146]: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-3 2024-09-13T01:00:09.528 INFO:journalctl@ceph.osd.3.smithi077.stdout:Sep 13 01:00:09 smithi077 bash[124207]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-3 2024-09-13T01:00:09.602 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:09 smithi088 bash[74473]: debug 2024-09-13T01:00:09.489+0000 7f01765f1640 -1 osd.4 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:40.186921+0000 front 2024-09-13T00:59:40.186555+0000 (oldest deadline 2024-09-13T01:00:03.086479+0000) 2024-09-13T01:00:09.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:09 smithi088 bash[45092]: debug 2024-09-13T01:00:09.265+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:10.103 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:09 smithi088 bash[33847]: debug 2024-09-13T01:00:09.761+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:10.103 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:09 smithi088 bash[39322]: debug 2024-09-13T01:00:09.905+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:10.524 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:10 smithi077 bash[72541]: ts=2024-09-13T01:00:10.205Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused" 2024-09-13T01:00:10.524 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:10 smithi077 bash[72541]: ts=2024-09-13T01:00:10.206Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused" 2024-09-13T01:00:10.524 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:10 smithi077 bash[72541]: ts=2024-09-13T01:00:10.207Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused" 2024-09-13T01:00:10.540 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:10 smithi088 bash[45092]: debug 2024-09-13T01:00:10.277+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:10.852 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:10 smithi088 bash[74473]: debug 2024-09-13T01:00:10.537+0000 7f01765f1640 -1 osd.4 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:40.186921+0000 front 2024-09-13T00:59:40.186555+0000 (oldest deadline 2024-09-13T01:00:03.086479+0000) 2024-09-13T01:00:10.853 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:10 smithi088 bash[33847]: debug 2024-09-13T01:00:10.769+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:11.169 DEBUG:teuthology.orchestra.run.smithi077:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.3.service' 2024-09-13T01:00:11.196 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T01:00:11.197 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-13T01:00:11.197 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-13T01:00:11.197 DEBUG:teuthology.orchestra.run.smithi088:> sudo systemctl stop ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.4 2024-09-13T01:00:11.208 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:10 smithi088 bash[39322]: debug 2024-09-13T01:00:10.881+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:11.576 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:11 smithi088 bash[45092]: debug 2024-09-13T01:00:11.265+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:11.576 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:11 smithi088 systemd[1]: Stopping Ceph osd.4 for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T01:00:11.576 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:11 smithi088 bash[74473]: debug 2024-09-13T01:00:11.345+0000 7f0179fdd640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T01:00:11.577 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:11 smithi088 bash[74473]: debug 2024-09-13T01:00:11.345+0000 7f0179fdd640 -1 osd.4 956 *** Got signal Terminated *** 2024-09-13T01:00:11.577 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:11 smithi088 bash[74473]: debug 2024-09-13T01:00:11.345+0000 7f0179fdd640 -1 osd.4 956 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T01:00:11.852 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:11 smithi088 bash[74473]: debug 2024-09-13T01:00:11.573+0000 7f01765f1640 -1 osd.4 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:40.186921+0000 front 2024-09-13T00:59:40.186555+0000 (oldest deadline 2024-09-13T01:00:03.086479+0000) 2024-09-13T01:00:11.853 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:11 smithi088 bash[33847]: debug 2024-09-13T01:00:11.781+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:12.262 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:11 smithi088 bash[39322]: debug 2024-09-13T01:00:11.877+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:12.579 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:12 smithi088 bash[45092]: debug 2024-09-13T01:00:12.257+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:12.852 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:12 smithi088 bash[74473]: debug 2024-09-13T01:00:12.577+0000 7f01765f1640 -1 osd.4 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:40.186921+0000 front 2024-09-13T00:59:40.186555+0000 (oldest deadline 2024-09-13T01:00:03.086479+0000) 2024-09-13T01:00:12.853 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:12 smithi088 bash[33847]: debug 2024-09-13T01:00:12.793+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:13.294 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:12 smithi088 bash[39322]: debug 2024-09-13T01:00:12.917+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:13.295 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:12 smithi088 bash[39322]: debug 2024-09-13T01:00:12.917+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:13.546 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:13 smithi088 bash[45092]: debug 2024-09-13T01:00:13.289+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:13.547 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:13 smithi088 bash[45092]: debug 2024-09-13T01:00:13.289+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:13.853 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:13 smithi088 bash[74473]: debug 2024-09-13T01:00:13.541+0000 7f01765f1640 -1 osd.4 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:40.186921+0000 front 2024-09-13T00:59:40.186555+0000 (oldest deadline 2024-09-13T01:00:03.086479+0000) 2024-09-13T01:00:13.853 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:13 smithi088 bash[74473]: debug 2024-09-13T01:00:13.541+0000 7f01765f1640 -1 osd.4 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:48.387756+0000 front 2024-09-13T00:59:48.387791+0000 (oldest deadline 2024-09-13T01:00:12.988361+0000) 2024-09-13T01:00:13.853 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:13 smithi088 bash[33847]: debug 2024-09-13T01:00:13.769+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:13.853 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:13 smithi088 bash[33847]: debug 2024-09-13T01:00:13.769+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:48.599714+0000 front 2024-09-13T00:59:48.599542+0000 (oldest deadline 2024-09-13T01:00:13.299382+0000) 2024-09-13T01:00:14.310 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:13 smithi088 bash[39322]: debug 2024-09-13T01:00:13.881+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:14.310 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:13 smithi088 bash[39322]: debug 2024-09-13T01:00:13.881+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:14.603 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:14 smithi088 bash[74473]: debug 2024-09-13T01:00:14.501+0000 7f01765f1640 -1 osd.4 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:40.186921+0000 front 2024-09-13T00:59:40.186555+0000 (oldest deadline 2024-09-13T01:00:03.086479+0000) 2024-09-13T01:00:14.603 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:14 smithi088 bash[74473]: debug 2024-09-13T01:00:14.501+0000 7f01765f1640 -1 osd.4 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:48.387756+0000 front 2024-09-13T00:59:48.387791+0000 (oldest deadline 2024-09-13T01:00:12.988361+0000) 2024-09-13T01:00:14.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:14 smithi088 bash[45092]: debug 2024-09-13T01:00:14.305+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:14.604 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:14 smithi088 bash[45092]: debug 2024-09-13T01:00:14.305+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:15.103 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:14 smithi088 bash[33847]: debug 2024-09-13T01:00:14.753+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:15.103 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:14 smithi088 bash[33847]: debug 2024-09-13T01:00:14.753+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:48.599714+0000 front 2024-09-13T00:59:48.599542+0000 (oldest deadline 2024-09-13T01:00:13.299382+0000) 2024-09-13T01:00:15.103 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:14 smithi088 bash[39322]: debug 2024-09-13T01:00:14.853+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:15.104 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:14 smithi088 bash[39322]: debug 2024-09-13T01:00:14.853+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:15.604 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:15 smithi088 bash[74473]: debug 2024-09-13T01:00:15.549+0000 7f01765f1640 -1 osd.4 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:40.186921+0000 front 2024-09-13T00:59:40.186555+0000 (oldest deadline 2024-09-13T01:00:03.086479+0000) 2024-09-13T01:00:15.605 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:15 smithi088 bash[74473]: debug 2024-09-13T01:00:15.549+0000 7f01765f1640 -1 osd.4 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:48.387756+0000 front 2024-09-13T00:59:48.387791+0000 (oldest deadline 2024-09-13T01:00:12.988361+0000) 2024-09-13T01:00:15.605 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:15 smithi088 bash[45092]: debug 2024-09-13T01:00:15.337+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:15.606 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:15 smithi088 bash[45092]: debug 2024-09-13T01:00:15.337+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:16.103 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:15 smithi088 bash[33847]: debug 2024-09-13T01:00:15.705+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:16.103 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:15 smithi088 bash[33847]: debug 2024-09-13T01:00:15.705+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:48.599714+0000 front 2024-09-13T00:59:48.599542+0000 (oldest deadline 2024-09-13T01:00:13.299382+0000) 2024-09-13T01:00:16.104 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:15 smithi088 bash[39322]: debug 2024-09-13T01:00:15.897+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:16.104 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:15 smithi088 bash[39322]: debug 2024-09-13T01:00:15.897+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:16.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:16 smithi088 bash[45092]: debug 2024-09-13T01:00:16.341+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:16.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:16 smithi088 bash[45092]: debug 2024-09-13T01:00:16.341+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:17.102 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:16 smithi088 bash[33847]: debug 2024-09-13T01:00:16.725+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:17.103 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:16 smithi088 bash[33847]: debug 2024-09-13T01:00:16.725+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:48.599714+0000 front 2024-09-13T00:59:48.599542+0000 (oldest deadline 2024-09-13T01:00:13.299382+0000) 2024-09-13T01:00:17.103 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:16 smithi088 bash[39322]: debug 2024-09-13T01:00:16.945+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:17.103 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:16 smithi088 bash[39322]: debug 2024-09-13T01:00:16.945+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:17.493 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:17 smithi088 bash[82115]: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-4 2024-09-13T01:00:17.494 INFO:journalctl@ceph.osd.4.smithi088.stdout:Sep 13 01:00:17 smithi088 bash[82182]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-4 2024-09-13T01:00:17.494 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:17 smithi088 bash[45092]: debug 2024-09-13T01:00:17.333+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:17.494 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:17 smithi088 bash[45092]: debug 2024-09-13T01:00:17.333+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:17.852 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:17 smithi088 bash[33847]: debug 2024-09-13T01:00:17.705+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:17.853 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:17 smithi088 bash[33847]: debug 2024-09-13T01:00:17.705+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:48.599714+0000 front 2024-09-13T00:59:48.599542+0000 (oldest deadline 2024-09-13T01:00:13.299382+0000) 2024-09-13T01:00:18.312 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:17 smithi088 bash[39322]: debug 2024-09-13T01:00:17.961+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:18.312 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:17 smithi088 bash[39322]: debug 2024-09-13T01:00:17.961+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:18.602 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:18 smithi088 bash[45092]: debug 2024-09-13T01:00:18.309+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:18.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:18 smithi088 bash[45092]: debug 2024-09-13T01:00:18.309+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:18.971 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:18 smithi088 bash[33847]: debug 2024-09-13T01:00:18.665+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:18.971 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:18 smithi088 bash[33847]: debug 2024-09-13T01:00:18.665+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:48.599714+0000 front 2024-09-13T00:59:48.599542+0000 (oldest deadline 2024-09-13T01:00:13.299382+0000) 2024-09-13T01:00:19.259 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:18 smithi088 bash[39322]: debug 2024-09-13T01:00:18.969+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:19.259 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:18 smithi088 bash[39322]: debug 2024-09-13T01:00:18.969+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:19.262 DEBUG:teuthology.orchestra.run.smithi088:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.4.service' 2024-09-13T01:00:19.331 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T01:00:19.331 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-13T01:00:19.331 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-13T01:00:19.332 DEBUG:teuthology.orchestra.run.smithi088:> sudo systemctl stop ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.5 2024-09-13T01:00:19.530 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:19 smithi088 systemd[1]: Stopping Ceph osd.5 for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T01:00:19.531 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:19 smithi088 bash[45092]: debug 2024-09-13T01:00:19.361+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:19.531 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:19 smithi088 bash[45092]: debug 2024-09-13T01:00:19.361+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:19.853 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:19 smithi088 bash[33847]: debug 2024-09-13T01:00:19.529+0000 7f261d7b9640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T01:00:19.853 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:19 smithi088 bash[33847]: debug 2024-09-13T01:00:19.529+0000 7f261d7b9640 -1 osd.5 956 *** Got signal Terminated *** 2024-09-13T01:00:19.853 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:19 smithi088 bash[33847]: debug 2024-09-13T01:00:19.529+0000 7f261d7b9640 -1 osd.5 956 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T01:00:19.853 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:19 smithi088 bash[33847]: debug 2024-09-13T01:00:19.669+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:19.853 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:19 smithi088 bash[33847]: debug 2024-09-13T01:00:19.669+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:48.599714+0000 front 2024-09-13T00:59:48.599542+0000 (oldest deadline 2024-09-13T01:00:13.299382+0000) 2024-09-13T01:00:20.320 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:20 smithi088 bash[39322]: debug 2024-09-13T01:00:20.017+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:20.321 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:20 smithi088 bash[39322]: debug 2024-09-13T01:00:20.017+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:20.528 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:20 smithi077 bash[72541]: ts=2024-09-13T01:00:20.206Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused" 2024-09-13T01:00:20.529 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:20 smithi077 bash[72541]: ts=2024-09-13T01:00:20.207Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused" 2024-09-13T01:00:20.529 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:20 smithi077 bash[72541]: ts=2024-09-13T01:00:20.208Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused" 2024-09-13T01:00:20.602 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:20 smithi088 bash[45092]: debug 2024-09-13T01:00:20.321+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:20.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:20 smithi088 bash[45092]: debug 2024-09-13T01:00:20.321+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:20.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:20 smithi088 bash[45092]: debug 2024-09-13T01:00:20.321+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:21.046 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:20 smithi088 bash[33847]: debug 2024-09-13T01:00:20.633+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:21.046 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:20 smithi088 bash[33847]: debug 2024-09-13T01:00:20.633+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:48.599714+0000 front 2024-09-13T00:59:48.599542+0000 (oldest deadline 2024-09-13T01:00:13.299382+0000) 2024-09-13T01:00:21.352 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:21 smithi088 bash[45092]: debug 2024-09-13T01:00:21.289+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:21.353 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:21 smithi088 bash[45092]: debug 2024-09-13T01:00:21.289+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:21.353 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:21 smithi088 bash[45092]: debug 2024-09-13T01:00:21.289+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:21.353 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:21 smithi088 bash[39322]: debug 2024-09-13T01:00:21.045+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:21.353 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:21 smithi088 bash[39322]: debug 2024-09-13T01:00:21.045+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:21.353 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:21 smithi088 bash[39322]: debug 2024-09-13T01:00:21.045+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:58.556281+0000 front 2024-09-13T00:59:58.556143+0000 (oldest deadline 2024-09-13T01:00:20.255924+0000) 2024-09-13T01:00:22.049 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:21 smithi088 bash[33847]: debug 2024-09-13T01:00:21.653+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:22.050 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:21 smithi088 bash[33847]: debug 2024-09-13T01:00:21.653+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:48.599714+0000 front 2024-09-13T00:59:48.599542+0000 (oldest deadline 2024-09-13T01:00:13.299382+0000) 2024-09-13T01:00:22.050 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:21 smithi088 bash[61791]: ts=2024-09-13T01:00:21.662Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.77:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.77:8765: connect: connection refused" 2024-09-13T01:00:22.050 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:21 smithi088 bash[61791]: ts=2024-09-13T01:00:21.665Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.77:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.77:8765: connect: connection refused" 2024-09-13T01:00:22.050 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:21 smithi088 bash[61791]: ts=2024-09-13T01:00:21.670Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.77:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.77:8765: connect: connection refused" 2024-09-13T01:00:22.050 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:21 smithi088 bash[61791]: ts=2024-09-13T01:00:21.670Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.77:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.77:8765: connect: connection refused" 2024-09-13T01:00:22.051 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:21 smithi088 bash[61791]: ts=2024-09-13T01:00:21.670Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.77:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.77:8765: connect: connection refused" 2024-09-13T01:00:22.305 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:22 smithi088 bash[39322]: debug 2024-09-13T01:00:22.049+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:22.306 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:22 smithi088 bash[39322]: debug 2024-09-13T01:00:22.049+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:22.306 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:22 smithi088 bash[39322]: debug 2024-09-13T01:00:22.049+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:58.556281+0000 front 2024-09-13T00:59:58.556143+0000 (oldest deadline 2024-09-13T01:00:20.255924+0000) 2024-09-13T01:00:22.602 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:22 smithi088 bash[45092]: debug 2024-09-13T01:00:22.305+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:22.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:22 smithi088 bash[45092]: debug 2024-09-13T01:00:22.305+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:22.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:22 smithi088 bash[45092]: debug 2024-09-13T01:00:22.305+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:23.085 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:22 smithi088 bash[33847]: debug 2024-09-13T01:00:22.649+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:23.085 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:22 smithi088 bash[33847]: debug 2024-09-13T01:00:22.649+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:48.599714+0000 front 2024-09-13T00:59:48.599542+0000 (oldest deadline 2024-09-13T01:00:13.299382+0000) 2024-09-13T01:00:23.086 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:22 smithi088 bash[33847]: debug 2024-09-13T01:00:22.649+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.002059+0000 front 2024-09-13T00:59:59.002105+0000 (oldest deadline 2024-09-13T01:00:22.501829+0000) 2024-09-13T01:00:23.353 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:23 smithi088 bash[45092]: debug 2024-09-13T01:00:23.325+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:23.353 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:23 smithi088 bash[45092]: debug 2024-09-13T01:00:23.325+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:23.353 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:23 smithi088 bash[45092]: debug 2024-09-13T01:00:23.325+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:23.354 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:23 smithi088 bash[39322]: debug 2024-09-13T01:00:23.085+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:23.354 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:23 smithi088 bash[39322]: debug 2024-09-13T01:00:23.085+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:23.354 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:23 smithi088 bash[39322]: debug 2024-09-13T01:00:23.085+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:58.556281+0000 front 2024-09-13T00:59:58.556143+0000 (oldest deadline 2024-09-13T01:00:20.255924+0000) 2024-09-13T01:00:24.103 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:23 smithi088 bash[33847]: debug 2024-09-13T01:00:23.645+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:42.198477+0000 front 2024-09-13T00:59:42.198191+0000 (oldest deadline 2024-09-13T01:00:03.897972+0000) 2024-09-13T01:00:24.103 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:23 smithi088 bash[33847]: debug 2024-09-13T01:00:23.645+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:48.599714+0000 front 2024-09-13T00:59:48.599542+0000 (oldest deadline 2024-09-13T01:00:13.299382+0000) 2024-09-13T01:00:24.103 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:23 smithi088 bash[33847]: debug 2024-09-13T01:00:23.645+0000 7f261add3640 -1 osd.5 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.002059+0000 front 2024-09-13T00:59:59.002105+0000 (oldest deadline 2024-09-13T01:00:22.501829+0000) 2024-09-13T01:00:24.603 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:24 smithi088 bash[39322]: debug 2024-09-13T01:00:24.125+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:24.603 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:24 smithi088 bash[39322]: debug 2024-09-13T01:00:24.125+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:24.603 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:24 smithi088 bash[39322]: debug 2024-09-13T01:00:24.125+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:58.556281+0000 front 2024-09-13T00:59:58.556143+0000 (oldest deadline 2024-09-13T01:00:20.255924+0000) 2024-09-13T01:00:24.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:24 smithi088 bash[45092]: debug 2024-09-13T01:00:24.313+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:24.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:24 smithi088 bash[45092]: debug 2024-09-13T01:00:24.313+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:24.604 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:24 smithi088 bash[45092]: debug 2024-09-13T01:00:24.313+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:25.270 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:25 smithi088 bash[39322]: debug 2024-09-13T01:00:25.149+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:25.270 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:25 smithi088 bash[39322]: debug 2024-09-13T01:00:25.149+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:25.270 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:25 smithi088 bash[39322]: debug 2024-09-13T01:00:25.149+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:58.556281+0000 front 2024-09-13T00:59:58.556143+0000 (oldest deadline 2024-09-13T01:00:20.255924+0000) 2024-09-13T01:00:25.602 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:25 smithi088 bash[82313]: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-5 2024-09-13T01:00:25.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:25 smithi088 bash[45092]: debug 2024-09-13T01:00:25.269+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:25.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:25 smithi088 bash[45092]: debug 2024-09-13T01:00:25.269+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:25.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:25 smithi088 bash[45092]: debug 2024-09-13T01:00:25.269+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:26.102 INFO:journalctl@ceph.osd.5.smithi088.stdout:Sep 13 01:00:25 smithi088 bash[82378]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-5 2024-09-13T01:00:26.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:26 smithi088 bash[45092]: debug 2024-09-13T01:00:26.297+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:26.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:26 smithi088 bash[45092]: debug 2024-09-13T01:00:26.297+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:26.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:26 smithi088 bash[45092]: debug 2024-09-13T01:00:26.297+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:26.603 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:26 smithi088 bash[39322]: debug 2024-09-13T01:00:26.169+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:26.603 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:26 smithi088 bash[39322]: debug 2024-09-13T01:00:26.169+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:26.604 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:26 smithi088 bash[39322]: debug 2024-09-13T01:00:26.169+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:58.556281+0000 front 2024-09-13T00:59:58.556143+0000 (oldest deadline 2024-09-13T01:00:20.255924+0000) 2024-09-13T01:00:27.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:27 smithi088 bash[45092]: debug 2024-09-13T01:00:27.325+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:27.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:27 smithi088 bash[45092]: debug 2024-09-13T01:00:27.325+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:27.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:27 smithi088 bash[45092]: debug 2024-09-13T01:00:27.325+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:27.603 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:27 smithi088 bash[39322]: debug 2024-09-13T01:00:27.177+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:27.603 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:27 smithi088 bash[39322]: debug 2024-09-13T01:00:27.177+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:27.604 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:27 smithi088 bash[39322]: debug 2024-09-13T01:00:27.177+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:58.556281+0000 front 2024-09-13T00:59:58.556143+0000 (oldest deadline 2024-09-13T01:00:20.255924+0000) 2024-09-13T01:00:27.748 DEBUG:teuthology.orchestra.run.smithi088:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.5.service' 2024-09-13T01:00:27.775 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T01:00:27.776 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-13T01:00:27.776 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-13T01:00:27.776 DEBUG:teuthology.orchestra.run.smithi088:> sudo systemctl stop ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.6 2024-09-13T01:00:28.103 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:27 smithi088 systemd[1]: Stopping Ceph osd.6 for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T01:00:28.103 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:27 smithi088 bash[39322]: debug 2024-09-13T01:00:27.953+0000 7f797bb3b640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T01:00:28.103 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:27 smithi088 bash[39322]: debug 2024-09-13T01:00:27.953+0000 7f797bb3b640 -1 osd.6 956 *** Got signal Terminated *** 2024-09-13T01:00:28.104 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:27 smithi088 bash[39322]: debug 2024-09-13T01:00:27.953+0000 7f797bb3b640 -1 osd.6 956 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T01:00:28.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:28 smithi088 bash[45092]: debug 2024-09-13T01:00:28.277+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:28.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:28 smithi088 bash[45092]: debug 2024-09-13T01:00:28.277+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:28.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:28 smithi088 bash[45092]: debug 2024-09-13T01:00:28.277+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:28.603 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:28 smithi088 bash[39322]: debug 2024-09-13T01:00:28.169+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:28.603 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:28 smithi088 bash[39322]: debug 2024-09-13T01:00:28.169+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:28.603 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:28 smithi088 bash[39322]: debug 2024-09-13T01:00:28.169+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:58.556281+0000 front 2024-09-13T00:59:58.556143+0000 (oldest deadline 2024-09-13T01:00:20.255924+0000) 2024-09-13T01:00:29.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:29 smithi088 bash[45092]: debug 2024-09-13T01:00:29.293+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:29.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:29 smithi088 bash[45092]: debug 2024-09-13T01:00:29.293+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:29.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:29 smithi088 bash[45092]: debug 2024-09-13T01:00:29.293+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:29.603 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:29 smithi088 bash[39322]: debug 2024-09-13T01:00:29.169+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:29.603 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:29 smithi088 bash[39322]: debug 2024-09-13T01:00:29.169+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:29.604 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:29 smithi088 bash[39322]: debug 2024-09-13T01:00:29.169+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:58.556281+0000 front 2024-09-13T00:59:58.556143+0000 (oldest deadline 2024-09-13T01:00:20.255924+0000) 2024-09-13T01:00:30.528 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:30 smithi077 bash[72541]: ts=2024-09-13T01:00:30.207Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused" 2024-09-13T01:00:30.529 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:30 smithi077 bash[72541]: ts=2024-09-13T01:00:30.208Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused" 2024-09-13T01:00:30.529 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:30 smithi077 bash[72541]: ts=2024-09-13T01:00:30.209Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused" 2024-09-13T01:00:30.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:30 smithi088 bash[45092]: debug 2024-09-13T01:00:30.249+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:30.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:30 smithi088 bash[45092]: debug 2024-09-13T01:00:30.249+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:30.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:30 smithi088 bash[45092]: debug 2024-09-13T01:00:30.249+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:30.603 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:30 smithi088 bash[39322]: debug 2024-09-13T01:00:30.165+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:30.603 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:30 smithi088 bash[39322]: debug 2024-09-13T01:00:30.165+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:30.604 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:30 smithi088 bash[39322]: debug 2024-09-13T01:00:30.165+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:58.556281+0000 front 2024-09-13T00:59:58.556143+0000 (oldest deadline 2024-09-13T01:00:20.255924+0000) 2024-09-13T01:00:31.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:31 smithi088 bash[45092]: debug 2024-09-13T01:00:31.225+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:31.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:31 smithi088 bash[45092]: debug 2024-09-13T01:00:31.225+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:31.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:31 smithi088 bash[45092]: debug 2024-09-13T01:00:31.225+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:31.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:31 smithi088 bash[45092]: debug 2024-09-13T01:00:31.225+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6830 osd.3 since back 2024-09-13T01:00:04.978822+0000 front 2024-09-13T01:00:04.978901+0000 (oldest deadline 2024-09-13T01:00:30.878559+0000) 2024-09-13T01:00:31.604 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:31 smithi088 bash[39322]: debug 2024-09-13T01:00:31.185+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:31.604 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:31 smithi088 bash[39322]: debug 2024-09-13T01:00:31.185+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:31.604 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:31 smithi088 bash[39322]: debug 2024-09-13T01:00:31.185+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:58.556281+0000 front 2024-09-13T00:59:58.556143+0000 (oldest deadline 2024-09-13T01:00:20.255924+0000) 2024-09-13T01:00:31.604 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:31 smithi088 bash[39322]: debug 2024-09-13T01:00:31.185+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6830 osd.3 since back 2024-09-13T01:00:06.057848+0000 front 2024-09-13T01:00:06.057712+0000 (oldest deadline 2024-09-13T01:00:30.757439+0000) 2024-09-13T01:00:32.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:32 smithi088 bash[45092]: debug 2024-09-13T01:00:32.209+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:32.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:32 smithi088 bash[45092]: debug 2024-09-13T01:00:32.209+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:32.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:32 smithi088 bash[45092]: debug 2024-09-13T01:00:32.209+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:32.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:32 smithi088 bash[45092]: debug 2024-09-13T01:00:32.209+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6830 osd.3 since back 2024-09-13T01:00:04.978822+0000 front 2024-09-13T01:00:04.978901+0000 (oldest deadline 2024-09-13T01:00:30.878559+0000) 2024-09-13T01:00:32.604 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:32 smithi088 bash[39322]: debug 2024-09-13T01:00:32.189+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:38.053371+0000 front 2024-09-13T00:59:38.053138+0000 (oldest deadline 2024-09-13T01:00:06.853803+0000) 2024-09-13T01:00:32.604 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:32 smithi088 bash[39322]: debug 2024-09-13T01:00:32.189+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:46.854870+0000 front 2024-09-13T00:59:46.854737+0000 (oldest deadline 2024-09-13T01:00:12.754363+0000) 2024-09-13T01:00:32.604 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:32 smithi088 bash[39322]: debug 2024-09-13T01:00:32.189+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:58.556281+0000 front 2024-09-13T00:59:58.556143+0000 (oldest deadline 2024-09-13T01:00:20.255924+0000) 2024-09-13T01:00:32.604 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:32 smithi088 bash[39322]: debug 2024-09-13T01:00:32.189+0000 7f7976950640 -1 osd.6 956 heartbeat_check: no reply from 172.21.15.77:6830 osd.3 since back 2024-09-13T01:00:06.057848+0000 front 2024-09-13T01:00:06.057712+0000 (oldest deadline 2024-09-13T01:00:30.757439+0000) 2024-09-13T01:00:33.526 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:33 smithi088 bash[45092]: debug 2024-09-13T01:00:33.181+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:33.527 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:33 smithi088 bash[45092]: debug 2024-09-13T01:00:33.181+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:33.527 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:33 smithi088 bash[45092]: debug 2024-09-13T01:00:33.181+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:33.527 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:33 smithi088 bash[45092]: debug 2024-09-13T01:00:33.181+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6830 osd.3 since back 2024-09-13T01:00:04.978822+0000 front 2024-09-13T01:00:04.978901+0000 (oldest deadline 2024-09-13T01:00:30.878559+0000) 2024-09-13T01:00:34.353 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:34 smithi088 bash[82511]: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-6 2024-09-13T01:00:34.353 INFO:journalctl@ceph.osd.6.smithi088.stdout:Sep 13 01:00:34 smithi088 bash[82573]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-6 2024-09-13T01:00:34.353 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:34 smithi088 bash[45092]: debug 2024-09-13T01:00:34.197+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:34.353 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:34 smithi088 bash[45092]: debug 2024-09-13T01:00:34.197+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:34.354 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:34 smithi088 bash[45092]: debug 2024-09-13T01:00:34.197+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:34.354 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:34 smithi088 bash[45092]: debug 2024-09-13T01:00:34.197+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6830 osd.3 since back 2024-09-13T01:00:04.978822+0000 front 2024-09-13T01:00:04.978901+0000 (oldest deadline 2024-09-13T01:00:30.878559+0000) 2024-09-13T01:00:35.473 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:35 smithi088 bash[45092]: debug 2024-09-13T01:00:35.149+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:35.473 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:35 smithi088 bash[45092]: debug 2024-09-13T01:00:35.149+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:35.473 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:35 smithi088 bash[45092]: debug 2024-09-13T01:00:35.149+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:35.474 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:35 smithi088 bash[45092]: debug 2024-09-13T01:00:35.149+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6830 osd.3 since back 2024-09-13T01:00:04.978822+0000 front 2024-09-13T01:00:04.978901+0000 (oldest deadline 2024-09-13T01:00:30.878559+0000) 2024-09-13T01:00:36.083 DEBUG:teuthology.orchestra.run.smithi088:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.6.service' 2024-09-13T01:00:36.110 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T01:00:36.111 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-13T01:00:36.111 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-13T01:00:36.111 DEBUG:teuthology.orchestra.run.smithi088:> sudo systemctl stop ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.7 2024-09-13T01:00:36.353 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:36 smithi088 systemd[1]: Stopping Ceph osd.7 for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T01:00:36.353 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:36 smithi088 bash[45092]: debug 2024-09-13T01:00:36.185+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:36.353 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:36 smithi088 bash[45092]: debug 2024-09-13T01:00:36.185+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:36.353 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:36 smithi088 bash[45092]: debug 2024-09-13T01:00:36.185+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:36.354 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:36 smithi088 bash[45092]: debug 2024-09-13T01:00:36.185+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6830 osd.3 since back 2024-09-13T01:00:04.978822+0000 front 2024-09-13T01:00:04.978901+0000 (oldest deadline 2024-09-13T01:00:30.878559+0000) 2024-09-13T01:00:36.354 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:36 smithi088 bash[45092]: debug 2024-09-13T01:00:36.297+0000 7f785ade5640 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-09-13T01:00:36.354 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:36 smithi088 bash[45092]: debug 2024-09-13T01:00:36.297+0000 7f785ade5640 -1 osd.7 956 *** Got signal Terminated *** 2024-09-13T01:00:36.354 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:36 smithi088 bash[45092]: debug 2024-09-13T01:00:36.297+0000 7f785ade5640 -1 osd.7 956 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-13T01:00:37.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:37 smithi088 bash[45092]: debug 2024-09-13T01:00:37.205+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:37.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:37 smithi088 bash[45092]: debug 2024-09-13T01:00:37.205+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:37.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:37 smithi088 bash[45092]: debug 2024-09-13T01:00:37.205+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:37.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:37 smithi088 bash[45092]: debug 2024-09-13T01:00:37.205+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6830 osd.3 since back 2024-09-13T01:00:04.978822+0000 front 2024-09-13T01:00:04.978901+0000 (oldest deadline 2024-09-13T01:00:30.878559+0000) 2024-09-13T01:00:37.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:37 smithi088 bash[45092]: debug 2024-09-13T01:00:37.205+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.88:6804 osd.4 since back 2024-09-13T01:00:15.480559+0000 front 2024-09-13T01:00:15.480655+0000 (oldest deadline 2024-09-13T01:00:37.180263+0000) 2024-09-13T01:00:38.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:38 smithi088 bash[45092]: debug 2024-09-13T01:00:38.157+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:38.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:38 smithi088 bash[45092]: debug 2024-09-13T01:00:38.157+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:38.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:38 smithi088 bash[45092]: debug 2024-09-13T01:00:38.157+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:38.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:38 smithi088 bash[45092]: debug 2024-09-13T01:00:38.157+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6830 osd.3 since back 2024-09-13T01:00:04.978822+0000 front 2024-09-13T01:00:04.978901+0000 (oldest deadline 2024-09-13T01:00:30.878559+0000) 2024-09-13T01:00:38.604 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:38 smithi088 bash[45092]: debug 2024-09-13T01:00:38.157+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.88:6804 osd.4 since back 2024-09-13T01:00:15.480559+0000 front 2024-09-13T01:00:15.480655+0000 (oldest deadline 2024-09-13T01:00:37.180263+0000) 2024-09-13T01:00:39.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:39 smithi088 bash[45092]: debug 2024-09-13T01:00:39.153+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:39.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:39 smithi088 bash[45092]: debug 2024-09-13T01:00:39.153+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:39.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:39 smithi088 bash[45092]: debug 2024-09-13T01:00:39.153+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:39.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:39 smithi088 bash[45092]: debug 2024-09-13T01:00:39.153+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6830 osd.3 since back 2024-09-13T01:00:04.978822+0000 front 2024-09-13T01:00:04.978901+0000 (oldest deadline 2024-09-13T01:00:30.878559+0000) 2024-09-13T01:00:39.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:39 smithi088 bash[45092]: debug 2024-09-13T01:00:39.153+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.88:6804 osd.4 since back 2024-09-13T01:00:15.480559+0000 front 2024-09-13T01:00:15.480655+0000 (oldest deadline 2024-09-13T01:00:37.180263+0000) 2024-09-13T01:00:40.528 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:40 smithi077 bash[72541]: ts=2024-09-13T01:00:40.208Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused" 2024-09-13T01:00:40.529 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:40 smithi077 bash[72541]: ts=2024-09-13T01:00:40.209Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused" 2024-09-13T01:00:40.529 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:40 smithi077 bash[72541]: ts=2024-09-13T01:00:40.210Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused" 2024-09-13T01:00:40.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:40 smithi088 bash[45092]: debug 2024-09-13T01:00:40.173+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:40.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:40 smithi088 bash[45092]: debug 2024-09-13T01:00:40.173+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:40.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:40 smithi088 bash[45092]: debug 2024-09-13T01:00:40.173+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:40.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:40 smithi088 bash[45092]: debug 2024-09-13T01:00:40.173+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6830 osd.3 since back 2024-09-13T01:00:04.978822+0000 front 2024-09-13T01:00:04.978901+0000 (oldest deadline 2024-09-13T01:00:30.878559+0000) 2024-09-13T01:00:40.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:40 smithi088 bash[45092]: debug 2024-09-13T01:00:40.173+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.88:6804 osd.4 since back 2024-09-13T01:00:15.480559+0000 front 2024-09-13T01:00:15.480655+0000 (oldest deadline 2024-09-13T01:00:37.180263+0000) 2024-09-13T01:00:41.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:41 smithi088 bash[45092]: debug 2024-09-13T01:00:41.189+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6806 osd.0 since back 2024-09-13T00:59:39.973657+0000 front 2024-09-13T00:59:39.974083+0000 (oldest deadline 2024-09-13T01:00:03.473744+0000) 2024-09-13T01:00:41.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:41 smithi088 bash[45092]: debug 2024-09-13T01:00:41.189+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6814 osd.1 since back 2024-09-13T00:59:49.275872+0000 front 2024-09-13T00:59:49.275838+0000 (oldest deadline 2024-09-13T01:00:12.775475+0000) 2024-09-13T01:00:41.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:41 smithi088 bash[45092]: debug 2024-09-13T01:00:41.189+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6822 osd.2 since back 2024-09-13T00:59:59.177447+0000 front 2024-09-13T00:59:59.177519+0000 (oldest deadline 2024-09-13T01:00:20.277331+0000) 2024-09-13T01:00:41.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:41 smithi088 bash[45092]: debug 2024-09-13T01:00:41.189+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.77:6830 osd.3 since back 2024-09-13T01:00:04.978822+0000 front 2024-09-13T01:00:04.978901+0000 (oldest deadline 2024-09-13T01:00:30.878559+0000) 2024-09-13T01:00:41.603 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:41 smithi088 bash[45092]: debug 2024-09-13T01:00:41.189+0000 7f7858c00640 -1 osd.7 956 heartbeat_check: no reply from 172.21.15.88:6804 osd.4 since back 2024-09-13T01:00:15.480559+0000 front 2024-09-13T01:00:15.480655+0000 (oldest deadline 2024-09-13T01:00:37.180263+0000) 2024-09-13T01:00:42.565 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:42 smithi088 bash[82703]: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-7 2024-09-13T01:00:42.566 INFO:journalctl@ceph.osd.7.smithi088.stdout:Sep 13 01:00:42 smithi088 bash[82765]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-osd-7 2024-09-13T01:00:44.167 DEBUG:teuthology.orchestra.run.smithi088:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@osd.7.service' 2024-09-13T01:00:44.194 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T01:00:44.194 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-13T01:00:44.195 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-13T01:00:44.195 DEBUG:teuthology.orchestra.run.smithi088:> sudo systemctl stop ceph-9218306c-7165-11ef-bceb-c7b262605968@prometheus.a 2024-09-13T01:00:44.548 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:44 smithi088 systemd[1]: Stopping Ceph prometheus.a for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T01:00:44.580 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:44 smithi088 bash[61791]: ts=2024-09-13T01:00:44.373Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-13T01:00:44.581 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:44 smithi088 bash[61791]: ts=2024-09-13T01:00:44.373Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-13T01:00:44.581 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:44 smithi088 bash[61791]: ts=2024-09-13T01:00:44.373Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-13T01:00:44.581 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:44 smithi088 bash[61791]: ts=2024-09-13T01:00:44.373Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-13T01:00:44.581 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:44 smithi088 bash[61791]: ts=2024-09-13T01:00:44.373Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-13T01:00:44.581 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:44 smithi088 bash[61791]: ts=2024-09-13T01:00:44.373Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-13T01:00:44.581 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:44 smithi088 bash[61791]: ts=2024-09-13T01:00:44.373Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-13T01:00:44.581 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:44 smithi088 bash[61791]: ts=2024-09-13T01:00:44.373Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-13T01:00:44.582 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:44 smithi088 bash[61791]: ts=2024-09-13T01:00:44.373Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-13T01:00:44.582 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:44 smithi088 bash[61791]: ts=2024-09-13T01:00:44.374Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-13T01:00:44.582 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:44 smithi088 bash[61791]: ts=2024-09-13T01:00:44.374Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-13T01:00:44.582 INFO:journalctl@ceph.prometheus.a.smithi088.stdout:Sep 13 01:00:44 smithi088 bash[61791]: ts=2024-09-13T01:00:44.374Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-13T01:00:44.985 DEBUG:teuthology.orchestra.run.smithi088:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9218306c-7165-11ef-bceb-c7b262605968@prometheus.a.service' 2024-09-13T01:00:45.007 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-13T01:00:45.008 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-13T01:00:45.008 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9218306c-7165-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T01:00:50.028 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 01:00:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:50.029 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:49 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:50.397 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 01:00:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:50.397 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:50.397 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:50 smithi077 bash[72541]: ts=2024-09-13T01:00:50.209Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused" 2024-09-13T01:00:50.398 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:50 smithi077 bash[72541]: ts=2024-09-13T01:00:50.210Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused" 2024-09-13T01:00:50.398 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:50 smithi077 bash[72541]: ts=2024-09-13T01:00:50.211Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused" 2024-09-13T01:00:50.659 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 01:00:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:50.659 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:50.910 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 01:00:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:50.910 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 01:00:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:50.910 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:51.205 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 01:00:51 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:51.205 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:50 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:51.206 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:51 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:51.467 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:51 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:51.467 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 01:00:51 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:51.764 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 01:00:51 smithi077 systemd[1]: Stopping Ceph node-exporter.a for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T01:00:51.765 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:51 smithi077 bash[72541]: ts=2024-09-13T01:00:51.627Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi077.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.77:8443: connect: connection refused" 2024-09-13T01:00:51.765 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:51 smithi077 bash[72541]: ts=2024-09-13T01:00:51.628Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi088.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.88:8443: connect: connection refused" 2024-09-13T01:00:52.486 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 01:00:52 smithi077 bash[124562]: ceph-9218306c-7165-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T01:00:52.487 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 01:00:52 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T01:00:52.487 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 01:00:52 smithi077 bash[124622]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-node-exporter-a 2024-09-13T01:00:52.487 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 01:00:52 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-13T01:00:52.487 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 01:00:52 smithi077 systemd[1]: Stopped Ceph node-exporter.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T01:00:52.745 INFO:journalctl@ceph.node-exporter.a.smithi077.stdout:Sep 13 01:00:52 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:52.745 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:52 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:52.745 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:52 smithi077 systemd[1]: Stopping Ceph alertmanager.a for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T01:00:52.745 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:52 smithi077 bash[72541]: ts=2024-09-13T01:00:52.700Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-13T01:00:53.506 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:53 smithi077 bash[124685]: ceph-9218306c-7165-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T01:00:53.507 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:53 smithi077 bash[124744]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-alertmanager-a 2024-09-13T01:00:53.507 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:53 smithi077 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-13T01:00:53.507 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:53 smithi077 systemd[1]: Stopped Ceph alertmanager.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T01:00:53.778 INFO:journalctl@ceph.alertmanager.a.smithi077.stdout:Sep 13 01:00:53 smithi077 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:00:56.520 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9218306c-7165-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-13T01:01:01.542 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 01:01:01 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:01.542 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 01:01:01 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:01.852 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 01:01:01 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:01.853 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 01:01:01 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:02.198 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 01:01:01 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:02.198 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 01:01:02 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:02.198 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 01:01:01 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:02.198 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 01:01:02 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:02.460 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 01:01:02 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:02.460 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 01:01:02 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:02.811 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 01:01:02 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:02.811 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 01:01:02 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:03.085 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 01:01:02 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:03.086 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 01:01:02 smithi088 systemd[1]: Stopping Ceph node-exporter.b for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T01:01:03.086 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 01:01:02 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:03.755 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 01:01:03 smithi088 bash[83220]: ceph-9218306c-7165-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T01:01:03.755 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 01:01:03 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-13T01:01:03.756 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 01:01:03 smithi088 bash[83279]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-node-exporter-b 2024-09-13T01:01:03.756 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 01:01:03 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-13T01:01:03.756 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 01:01:03 smithi088 systemd[1]: Stopped Ceph node-exporter.b for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T01:01:04.103 INFO:journalctl@ceph.node-exporter.b.smithi088.stdout:Sep 13 01:01:03 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:04.103 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 01:01:03 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:04.104 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 01:01:03 smithi088 systemd[1]: Stopping Ceph grafana.a for 9218306c-7165-11ef-bceb-c7b262605968... 2024-09-13T01:01:04.468 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 01:01:04 smithi088 bash[83343]: ceph-9218306c-7165-11ef-bceb-c7b262605968-grafana-a 2024-09-13T01:01:04.818 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 01:01:04 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-13T01:01:04.818 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 01:01:04 smithi088 bash[83402]: Error response from daemon: No such container: ceph-9218306c-7165-11ef-bceb-c7b262605968-grafana-a 2024-09-13T01:01:04.818 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 01:01:04 smithi088 systemd[1]: ceph-9218306c-7165-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-13T01:01:04.818 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 01:01:04 smithi088 systemd[1]: Stopped Ceph grafana.a for 9218306c-7165-11ef-bceb-c7b262605968. 2024-09-13T01:01:04.819 INFO:journalctl@ceph.grafana.a.smithi088.stdout:Sep 13 01:01:04 smithi088 systemd[1]: /etc/systemd/system/ceph-9218306c-7165-11ef-bceb-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-09-13T01:01:15.411 DEBUG:teuthology.orchestra.run.smithi077:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T01:01:15.425 DEBUG:teuthology.orchestra.run.smithi088:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-13T01:01:15.437 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-13T01:01:15.439 DEBUG:teuthology.misc:Transferring archived files from smithi077:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901110/remote/smithi077/crash 2024-09-13T01:01:15.440 DEBUG:teuthology.orchestra.run.smithi077:> sudo tar c -f - -C /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/crash -- . 2024-09-13T01:01:15.479 INFO:teuthology.orchestra.run.smithi077.stderr:tar: /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T01:01:15.479 INFO:teuthology.orchestra.run.smithi077.stderr:tar: Error is not recoverable: exiting now 2024-09-13T01:01:15.481 DEBUG:teuthology.misc:Transferring archived files from smithi088:/var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901110/remote/smithi088/crash 2024-09-13T01:01:15.481 DEBUG:teuthology.orchestra.run.smithi088:> sudo tar c -f - -C /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/crash -- . 2024-09-13T01:01:15.491 INFO:teuthology.orchestra.run.smithi088.stderr:tar: /var/lib/ceph/9218306c-7165-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-13T01:01:15.492 INFO:teuthology.orchestra.run.smithi088.stderr:tar: Error is not recoverable: exiting now 2024-09-13T01:01:15.493 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-13T01:01:15.493 DEBUG:teuthology.orchestra.run.smithi077:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/9218306c-7165-11ef-bceb-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-09-13T01:01:15.535 INFO:teuthology.orchestra.run.smithi077.stderr:grep: /var/log/ceph/9218306c-7165-11ef-bceb-c7b262605968/ceph.log: No such file or directory 2024-09-13T01:01:15.536 INFO:tasks.cephadm:Compressing logs... 2024-09-13T01:01:15.536 DEBUG:teuthology.orchestra.run.smithi077:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T01:01:15.581 DEBUG:teuthology.orchestra.run.smithi088:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T01:01:15.590 INFO:teuthology.orchestra.run.smithi077.stderr:find: '/var/log/rbd-target-api': No such file or directory 2024-09-13T01:01:15.592 INFO:teuthology.orchestra.run.smithi088.stderr:find: '/var/log/rbd-target-api': No such file or directory 2024-09-13T01:01:54.896 INFO:tasks.cephadm:Archiving logs... 2024-09-13T01:01:54.897 DEBUG:teuthology.misc:Transferring archived files from smithi077:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901110/remote/smithi077/log 2024-09-13T01:01:54.898 DEBUG:teuthology.orchestra.run.smithi077:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T01:01:58.613 DEBUG:teuthology.misc:Transferring archived files from smithi088:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901110/remote/smithi088/log 2024-09-13T01:01:58.614 DEBUG:teuthology.orchestra.run.smithi088:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-13T01:02:02.018 INFO:tasks.cephadm:Removing cluster... 2024-09-13T01:02:02.019 DEBUG:teuthology.orchestra.run.smithi077:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9218306c-7165-11ef-bceb-c7b262605968 --force 2024-09-13T01:02:03.571 DEBUG:teuthology.orchestra.run.smithi088:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9218306c-7165-11ef-bceb-c7b262605968 --force 2024-09-13T01:02:05.137 INFO:tasks.cephadm:Removing cephadm ... 2024-09-13T01:02:05.137 DEBUG:teuthology.orchestra.run.smithi077:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T01:02:05.143 DEBUG:teuthology.orchestra.run.smithi088:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-13T01:02:05.149 INFO:tasks.cephadm:Teardown complete 2024-09-13T01:02:05.149 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-13T01:02:05.161 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-13T01:02:05.161 DEBUG:teuthology.orchestra.run.smithi077:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-13T01:02:05.189 DEBUG:teuthology.orchestra.run.smithi088:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-13T01:02:05.250 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-13T01:02:05.259 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-13T01:02:05.260 DEBUG:teuthology.orchestra.run.smithi077:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T01:02:05.262 DEBUG:teuthology.orchestra.run.smithi088:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-13T01:02:05.272 INFO:teuthology.orchestra.run.smithi077.stdout: remote refid st t when poll reach delay offset jitter 2024-09-13T01:02:05.272 INFO:teuthology.orchestra.run.smithi077.stdout:============================================================================== 2024-09-13T01:02:05.272 INFO:teuthology.orchestra.run.smithi077.stdout:*hv01.front.sepi 67.205.162.81 3 u 14 64 377 0.098 -2.937 0.265 2024-09-13T01:02:05.272 INFO:teuthology.orchestra.run.smithi077.stdout:+hv02.front.sepi 63.231.80.2 3 u 11 64 377 0.083 -2.793 0.289 2024-09-13T01:02:05.273 INFO:teuthology.orchestra.run.smithi077.stdout:+hv03.front.sepi 74.6.168.72 3 u 7 64 377 0.095 -2.908 0.294 2024-09-13T01:02:05.273 INFO:teuthology.orchestra.run.smithi077.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-09-13T01:02:05.275 INFO:teuthology.orchestra.run.smithi088.stdout: remote refid st t when poll reach delay offset jitter 2024-09-13T01:02:05.275 INFO:teuthology.orchestra.run.smithi088.stdout:============================================================================== 2024-09-13T01:02:05.275 INFO:teuthology.orchestra.run.smithi088.stdout:+hv01.front.sepi 67.205.162.81 3 u 19 64 377 0.105 -2.062 0.136 2024-09-13T01:02:05.275 INFO:teuthology.orchestra.run.smithi088.stdout:+hv02.front.sepi 63.231.80.2 3 u 24 64 377 0.092 -1.729 0.157 2024-09-13T01:02:05.275 INFO:teuthology.orchestra.run.smithi088.stdout:*hv03.front.sepi 74.6.168.72 3 u 21 64 377 0.084 -2.201 0.119 2024-09-13T01:02:05.275 INFO:teuthology.orchestra.run.smithi088.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-09-13T01:02:05.276 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-13T01:02:05.285 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-13T01:02:05.286 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-13T01:02:05.295 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-13T01:02:05.306 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-13T01:02:05.315 INFO:teuthology.task.internal:Duration was 3222.845632 seconds 2024-09-13T01:02:05.315 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-13T01:02:05.325 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-13T01:02:05.325 DEBUG:teuthology.orchestra.run.smithi077:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T01:02:05.328 DEBUG:teuthology.orchestra.run.smithi088:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-13T01:02:05.363 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-13T01:02:05.363 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi077.front.sepia.ceph.com 2024-09-13T01:02:05.363 DEBUG:teuthology.orchestra.run.smithi077:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T01:02:05.413 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi088.front.sepia.ceph.com 2024-09-13T01:02:05.413 DEBUG:teuthology.orchestra.run.smithi088:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-13T01:02:05.423 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-13T01:02:05.424 DEBUG:teuthology.orchestra.run.smithi077:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T01:02:05.457 DEBUG:teuthology.orchestra.run.smithi088:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-13T01:02:05.530 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-13T01:02:05.530 DEBUG:teuthology.orchestra.run.smithi077:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T01:02:05.532 DEBUG:teuthology.orchestra.run.smithi088:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-13T01:02:05.737 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-13T01:02:05.747 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-13T01:02:05.748 DEBUG:teuthology.orchestra.run.smithi077:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T01:02:05.759 DEBUG:teuthology.orchestra.run.smithi088:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-13T01:02:05.772 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-13T01:02:05.782 DEBUG:teuthology.orchestra.run.smithi077:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T01:02:05.805 DEBUG:teuthology.orchestra.run.smithi088:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-13T01:02:05.814 INFO:teuthology.orchestra.run.smithi077.stdout:kernel.core_pattern = core 2024-09-13T01:02:05.826 INFO:teuthology.orchestra.run.smithi088.stdout:kernel.core_pattern = core 2024-09-13T01:02:05.851 DEBUG:teuthology.orchestra.run.smithi077:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T01:02:05.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T01:02:05.871 DEBUG:teuthology.orchestra.run.smithi088:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-13T01:02:05.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-13T01:02:05.900 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-13T01:02:05.911 INFO:teuthology.task.internal:Transferring archived files... 2024-09-13T01:02:05.911 DEBUG:teuthology.misc:Transferring archived files from smithi077:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901110/remote/smithi077 2024-09-13T01:02:05.911 DEBUG:teuthology.orchestra.run.smithi077:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T01:02:05.948 DEBUG:teuthology.misc:Transferring archived files from smithi088:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-11_01:16:05-upgrade:quincy-x-reef-distro-default-smithi/7901110/remote/smithi088 2024-09-13T01:02:05.948 DEBUG:teuthology.orchestra.run.smithi088:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-13T01:02:05.974 INFO:teuthology.task.internal:Removing archive directory... 2024-09-13T01:02:05.975 DEBUG:teuthology.orchestra.run.smithi077:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T01:02:05.981 DEBUG:teuthology.orchestra.run.smithi088:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-13T01:02:06.017 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-13T01:02:06.027 INFO:teuthology.task.internal:Not uploading archives. 2024-09-13T01:02:06.028 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-13T01:02:06.037 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-13T01:02:06.037 DEBUG:teuthology.orchestra.run.smithi077:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T01:02:06.039 DEBUG:teuthology.orchestra.run.smithi088:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-13T01:02:06.043 INFO:teuthology.orchestra.run.smithi077.stdout: 658640 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 01:02 /home/ubuntu/cephtest 2024-09-13T01:02:06.064 INFO:teuthology.orchestra.run.smithi088.stdout: 658640 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 13 01:02 /home/ubuntu/cephtest 2024-09-13T01:02:06.065 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-13T01:02:06.076 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-13T01:02:06.101 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 3222.845632314682 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=718e0974f06247e2a1fc22a5b88abc7c status: fail success: false 2024-09-13T01:02:06.101 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-13T01:02:06.178 INFO:teuthology.run:FAIL